From 8d3dd3aa02d3e31e544ae03014e9339159eb1a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 22 Nov 2024 16:38:23 +0100 Subject: [PATCH 01/47] Run testing workflow unconditionally --- .forgejo/workflows/testing.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 62136b1b28..9ba8c47a01 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -10,7 +10,6 @@ on: jobs: backend-checks: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker container: image: 'data.forgejo.org/oci/node:22-bookworm' @@ -27,7 +26,6 @@ jobs: - run: su forgejo -c 'make --always-make -j$(nproc) lint-backend tidy-check swagger-check lint-swagger fmt-check swagger-validate' # ensure the "go-licenses" make target runs - uses: ./.forgejo/workflows-composite/build-backend frontend-checks: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker container: image: 'data.forgejo.org/oci/node:22-bookworm' @@ -176,7 +174,6 @@ jobs: TAGS: bindata TEST_REDIS_SERVER: cacher:${{ matrix.cacher.port }} test-mysql: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: [backend-checks, frontend-checks] container: @@ -207,7 +204,6 @@ jobs: env: USE_REPO_TEST_DIR: 1 test-pgsql: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: [backend-checks, frontend-checks] container: @@ -246,7 +242,6 @@ jobs: USE_REPO_TEST_DIR: 1 TEST_LDAP: 1 test-sqlite: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: [backend-checks, frontend-checks] container: @@ -269,14 +264,11 @@ jobs: TEST_TAGS: sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 security-check: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: - test-sqlite - test-pgsql - test-mysql - - test-remote-cacher - - test-unit container: image: 'data.forgejo.org/oci/node:22-bookworm' options: --tmpfs /tmp:exec,noatime From 16c0285ca466b5c454fbacd34628e8e97d4a4a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 30 Apr 2024 12:33:18 +0200 Subject: [PATCH 02/47] Fix name in package-lock.json The frontend-checks job started failing because the declared name does not match the repository name. --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index dfd359b31b..318521d1ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,10 @@ { - "name": "forgejo", + "name": "forgejo-aneksajo", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "forgejo", + "name": "forgejo-aneksajo", "dependencies": { "@citation-js/core": "0.7.14", "@citation-js/plugin-bibtex": "0.7.16", diff --git a/package.json b/package.json index 75739b3be0..c79d20abbc 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "forgejo", + "name": "forgejo-aneksajo", "type": "module", "engines": { "node": ">= 20.0.0" From d9987529035a25736cb069fab7d92d3317997d14 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sat, 30 Jul 2022 21:45:03 -0400 Subject: [PATCH 03/47] git-annex support [git-annex](https://git-annex.branchable.com/) is a more complicated cousin to git-lfs, storing large files in an optional-download side content. Unlike lfs, it allows mixing and matching storage remotes, so the content remote(s) doesn't need to be on the same server as the git remote, making it feasible to scatter a collection across cloud storage, old harddrives, or anywhere else storage can be scavenged. Since this can get complicated, fast, it has a content-tracking database (`git annex whereis`) to help find everything later. The use-case we imagine for including it in Gitea is just the simple case, where we're primarily emulating git-lfs: each repo has its large content at the same URL. Our motivation is so we can self-host https://www.datalad.org/ datasets, which currently are only hostable by fragilely scrounging together cloud storage -- and having to manage all the credentials associated with all the pieces -- or at https://openneuro.org which is fragile in its own ways. Supporting git-annex also allows multiple Gitea instance to be annex remotes for each other, mirroring the content or otherwise collaborating the split up the hosting costs. Enabling -------- TODO HTTP ---- TODO Permission Checking ------------------- This tweaks the API in routers/private/serv.go to expose the calling user's computed permission, instead of just returning HTTP 403. This doesn't fit in super well. It's the opposite from how the git-lfs support is done, where there's a complete list of possible subcommands and their matching permission levels, and then the API compares the requested with the actual level and returns HTTP 403 if the check fails. But it's necessary. The main git-annex verbs, 'git-annex-shell configlist' and 'git-annex-shell p2pstdio' are both either read-only or read-write operations, depending on the state on disk on either end of the connection and what the user asked it to ask for, with no way to know before git-annex examines the situation. So tell the level via GIT_ANNEX_READONLY and trust it to handle itself. In the older Gogs version, the permission was directly read in cmd/serv.go: ``` mode, err = db.UserAccessMode(user.ID, repo) ``` - https://github.com/G-Node/gogs/blob/966e925cf320beff768b192276774d9265706df5/internal/cmd/serv.go#L334 but in Gitea permission enforcement has been centralized in the API layer. (perhaps so the cmd layer can avoid making direct DB connections?) Deletion -------- git-annex has this "lockdown" feature where it tries really quite very hard to prevent you deleting its data, to the point that even an rm -rf won't do it: each file in annex/objects/ is nested inside a folder with read-only permissions. The recommended workaround is to run chmod -R +w when you're sure you actually want to delete a repo. See https://git-annex.branchable.com/internals/lockdown So we edit util.RemoveAll() to do just that, so now it's `chmod -R +w && rm -rf` instead of just `rm -rf`. --- cmd/serv.go | 72 ++++++++++++++++++++++++++++++++++++++--- modules/private/serv.go | 1 + modules/util/remove.go | 34 ++++++++++++++++++- routers/private/serv.go | 12 +++++-- 4 files changed, 110 insertions(+), 9 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 0884d6c36b..59379a91b4 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -38,6 +38,7 @@ import ( const ( lfsAuthenticateVerb = "git-lfs-authenticate" + gitAnnexShellVerb = "git-annex-shell" ) // CmdServ represents the available serv sub-command. @@ -79,6 +80,7 @@ var ( "git-upload-archive": perm.AccessModeRead, "git-receive-pack": perm.AccessModeWrite, lfsAuthenticateVerb: perm.AccessModeNone, + gitAnnexShellVerb: perm.AccessModeNone, // annex permissions are enforced by GIT_ANNEX_SHELL_READONLY, rather than the Gitea API } alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -212,6 +214,29 @@ func runServ(c *cli.Context) error { } } + if verb == gitAnnexShellVerb { + // if !setting.Annex.Enabled { // TODO: https://github.com/neuropoly/gitea/issues/8 + if false { + return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") + } + + if len(words) < 3 { + return fail(ctx, "Too few arguments", "Too few arguments in cmd: %s", cmd) + } + + // git-annex always puts the repo in words[2], unlike most other + // git subcommands; and it sometimes names repos like /~/, as if + // $HOME should get expanded while also being rooted. e.g.: + // git-annex-shell 'configlist' '/~/user/repo' + // git-annex-shell 'sendkey' '/user/repo 'key' + repoPath = words[2] + repoPath = strings.TrimPrefix(repoPath, "/") + repoPath = strings.TrimPrefix(repoPath, "~/") + } + + // prevent directory traversal attacks + repoPath = filepath.Clean("/" + repoPath)[1:] + rr := strings.SplitN(repoPath, "/", 2) if len(rr) != 2 { return fail(ctx, "Invalid repository path", "Invalid repository path: %v", repoPath) @@ -225,6 +250,18 @@ func runServ(c *cli.Context) error { // so that username and reponame are not affected. repoPath = strings.ToLower(strings.TrimSpace(repoPath)) + // put the sanitized repoPath back into the argument list for later + if verb == gitAnnexShellVerb { + // git-annex-shell demands an absolute path + absRepoPath, err := filepath.Abs(filepath.Join(setting.RepoRootPath, repoPath)) + if err != nil { + return fail(ctx, "Error locating repoPath", "%v", err) + } + words[2] = absRepoPath + } else { + words[1] = repoPath + } + if alphaDashDotPattern.MatchString(reponame) { return fail(ctx, "Invalid repo name", "Invalid repo name: %s", reponame) } @@ -303,21 +340,46 @@ func runServ(c *cli.Context) error { return nil } - var gitcmd *exec.Cmd gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack if _, err := os.Stat(gitBinVerb); err != nil { // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way + // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath, + // but '{gitBinPath}/git annex-shell' should be able to find them on $PATH. verbFields := strings.SplitN(verb, "-", 2) if len(verbFields) == 2 { // use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ... - gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath) + gitBinVerb = git.GitExecutable + words = append([]string{verbFields[1]}, words...) } } - if gitcmd == nil { - // by default, use the verb (it has been checked above by allowedCommands) - gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath) + + // by default, use the verb (it has been checked above by allowedCommands) + gitcmd := exec.CommandContext(ctx, gitBinVerb, words[1:]...) + + if verb == gitAnnexShellVerb { + // This doesn't get its own isolated section like LFS does, because LFS + // is handled by internal Gitea routines, but git-annex has to be shelled out + // to like other git subcommands, so we need to build up gitcmd. + + // TODO: does this work on Windows? + gitcmd.Env = append(gitcmd.Env, + // "If set, disallows running git-shell to handle unknown commands." + // - git-annex-shell(1) + "GIT_ANNEX_SHELL_LIMITED=True", + // "If set, git-annex-shell will refuse to run commands + // that do not operate on the specified directory." + // - git-annex-shell(1) + fmt.Sprintf("GIT_ANNEX_SHELL_DIRECTORY=%s", words[2]), + ) + if results.UserMode < perm.AccessModeWrite { + // "If set, disallows any action that could modify the git-annex repository." + // - git-annex-shell(1) + // We set this when the backend API has told us that we don't have write permission to this repo. + log.Debug("Setting GIT_ANNEX_SHELL_READONLY=True") + gitcmd.Env = append(gitcmd.Env, "GIT_ANNEX_SHELL_READONLY=True") + } } process.SetSysProcAttribute(gitcmd) diff --git a/modules/private/serv.go b/modules/private/serv.go index fb8496930e..af4a016cb8 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -40,6 +40,7 @@ type ServCommandResults struct { UserName string UserEmail string UserID int64 + UserMode perm.AccessMode OwnerName string RepoName string RepoID int64 diff --git a/modules/util/remove.go b/modules/util/remove.go index 2a65a6b0aa..0d0323d0b4 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -4,7 +4,9 @@ package util import ( + "io/fs" "os" + "path/filepath" "syscall" "time" ) @@ -32,10 +34,40 @@ func Remove(name string) error { return err } -// RemoveAll removes the named file or (empty) directory with at most 5 attempts. +// RemoveAll removes the named file or directory with at most 5 attempts. func RemoveAll(name string) error { var err error + for i := 0; i < 5; i++ { + // Do chmod -R +w to help ensure the removal succeeds. + // In particular, in the git-annex case, this handles + // https://git-annex.branchable.com/internals/lockdown/ : + // + // > (The only bad consequence of this is that rm -rf .git + // > doesn't work unless you first run chmod -R +w .git) + + err = filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { + // NB: this is called WalkDir but it works on a single file too + if err == nil { + info, err := d.Info() + if err != nil { + return err + } + + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + // try again + <-time.After(100 * time.Millisecond) + continue + } + err = os.RemoveAll(name) if err == nil { break diff --git a/routers/private/serv.go b/routers/private/serv.go index df61355fb0..82953b5af9 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -81,12 +81,14 @@ func ServCommand(ctx *context.PrivateContext) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") mode := perm.AccessMode(ctx.FormInt("mode")) + verbs := ctx.FormStrings("verb") // Set the basic parts of the results to return results := private.ServCommandResults{ RepoName: repoName, OwnerName: ownerName, KeyID: keyID, + UserMode: perm.AccessModeNone, } // Now because we're not translating things properly let's just default some English strings here @@ -287,8 +289,10 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey + ( /*setting.Annex.Enabled && */ len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { + results.UserMode = deployKey.Mode if deployKey.Mode < mode { ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, results.OwnerName, results.RepoName), @@ -310,9 +314,9 @@ func ServCommand(ctx *context.PrivateContext) { return } - userMode := perm.UnitAccessMode(unitType) + results.UserMode = perm.UnitAccessMode(unitType) - if userMode < mode { + if results.UserMode < mode { log.Warn("Failed authentication attempt for %s with key %s (not authorized to %s %s/%s) from %s", user.Name, key.Name, modeString, ownerName, repoName, ctx.RemoteAddr()) ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("User: %d:%s with Key: %d:%s is not authorized to %s %s/%s.", user.ID, user.Name, key.ID, key.Name, modeString, ownerName, repoName), @@ -353,6 +357,7 @@ func ServCommand(ctx *context.PrivateContext) { }) return } + results.UserMode = perm.AccessModeWrite results.RepoID = repo.ID } @@ -381,13 +386,14 @@ func ServCommand(ctx *context.PrivateContext) { return } } - log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", + log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nUserMode: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", results.IsWiki, results.DeployKeyID, results.KeyID, results.KeyName, results.UserName, results.UserID, + results.UserMode, results.OwnerName, results.RepoName, results.RepoID) From 1d53f9f2cdcba63f7fd76ba4c27f4d876a03a48e Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 19 Aug 2022 02:49:18 -0400 Subject: [PATCH 04/47] git-annex tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/neuropoly/gitea/issues/11 Tests: * `git annex init` * `git annex copy --from origin` * `git annex copy --to origin` over: * ssh for: * the owner * a collaborator * a read-only collaborator * a stranger in a * public repo * private repo And then confirms: * Deletion of the remote repo (to ensure lockdown isn't messing with us: https://git-annex.branchable.com/internals/lockdown/#comment-0cc5225dc5abe8eddeb843bfd2fdc382) ------ To support all this: * Add util.FileCmp() * Patch withKeyFile() so it can be nested in other copies of itself ------- Many thanks to Mathieu for giving style tips and catching several bugs, including a subtle one in util.filecmp() which neutered it. Co-authored-by: Mathieu Guay-Paquet Co-authored-by: Matthias Riße --- Makefile | 2 +- modules/util/filecmp.go | 87 ++ modules/util/remove.go | 37 +- .../api_helper_for_declarative_test.go | 25 + tests/integration/git_annex_test.go | 760 ++++++++++++++++++ .../git_helper_for_declarative_test.go | 22 + 6 files changed, 916 insertions(+), 17 deletions(-) create mode 100644 modules/util/filecmp.go create mode 100644 tests/integration/git_annex_test.go diff --git a/Makefile b/Makefile index 4fb06db918..66f8524c04 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ self := $(location) @tmpdir=`mktemp --tmpdir -d` ; \ echo Using temporary directory $$tmpdir for test repositories ; \ USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \ - STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS + STATUS=$$? ; chmod -R +w "$$tmpdir" && rm -r "$$tmpdir" ; exit $$STATUS else diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go new file mode 100644 index 0000000000..76e7705cc1 --- /dev/null +++ b/modules/util/filecmp.go @@ -0,0 +1,87 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "bytes" + "io" + "os" +) + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file2) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} diff --git a/modules/util/remove.go b/modules/util/remove.go index 0d0323d0b4..4e5271136d 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -34,6 +34,26 @@ func Remove(name string) error { return err } +// MakeWritable recursively makes the named directory writable. +func MakeWritable(name string) error { + return filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { + // NB: this is called WalkDir but it works on a single file too + if err == nil { + info, err := d.Info() + if err != nil { + return err + } + + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } + } + return nil + }) +} + // RemoveAll removes the named file or directory with at most 5 attempts. func RemoveAll(name string) error { var err error @@ -46,22 +66,7 @@ func RemoveAll(name string) error { // > (The only bad consequence of this is that rm -rf .git // > doesn't work unless you first run chmod -R +w .git) - err = filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { - // NB: this is called WalkDir but it works on a single file too - if err == nil { - info, err := d.Info() - if err != nil { - return err - } - - // 0200 == u+w, in octal unix permission notation - err = os.Chmod(path, info.Mode()|0o200) - if err != nil { - return err - } - } - return nil - }) + err = MakeWritable(name) if err != nil { // try again <-time.After(100 * time.Millisecond) diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index b5941b3389..88e1cd2198 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -21,6 +21,7 @@ import ( api "forgejo.org/modules/structs" "forgejo.org/services/forms" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -460,3 +461,27 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +// generate and activate an ssh key for the user attached to the APITestContext +// TODO: pick a better name; golang doesn't do method overriding. +func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + // we need to have write:public_key to do this step + // the easiest way is to create a throwaway ctx that is identical but only has that permission + ctxKeyWriter := ctx + ctxKeyWriter.Token = getTokenForLoggedInUser(t, ctx.Session, auth.AccessTokenScopeWriteUser) + + keyName := "One of " + ctx.Username + "'s keys: #" + uuid.New().String() + withKeyFile(t, keyName, func(keyFile string) { + var key api.PublicKey + + doAPICreateUserKey(ctxKeyWriter, keyName, keyFile, + func(t *testing.T, _key api.PublicKey) { + // save the key ID so we can delete it at the end + key = _key + })(t) + + defer doAPIDeleteUserKey(ctxKeyWriter, key.ID)(t) + + callback() + }) +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go new file mode 100644 index 0000000000..f64b1d98df --- /dev/null +++ b/tests/integration/git_annex_test.go @@ -0,0 +1,760 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "errors" + "fmt" + "math/rand" + "net/url" + "os" + "path" + "regexp" + "strings" + "testing" + + auth_model "forgejo.org/models/auth" + "forgejo.org/models/db" + "forgejo.org/models/perm" + repo_model "forgejo.org/models/repo" + "forgejo.org/modules/git" + "forgejo.org/modules/setting" + api "forgejo.org/modules/structs" + "forgejo.org/modules/util" + "forgejo.org/tests" + + "github.com/stretchr/testify/require" +) + +// Some guidelines: +// +// * a APITestContext is an awkward union of session credential + username + target repo +// which is assumed to be owned by that username; if you want to target a different +// repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + +func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool) (err error) { + // creating a repo counts as editing the user's profile (is done by POSTing + // to /api/v1/user/repos/) -- which means it needs a User-scoped token and + // both that and editing need a Repo-scoped token because they edit repositories. + rescopedCtx := ctx + rescopedCtx.Token = getTokenForLoggedInUser(t, ctx.Session, auth_model.AccessTokenScopeWriteUser, auth_model.AccessTokenScopeWriteRepository) + doAPICreateRepository(rescopedCtx, false)(t) + t.Cleanup(func() { util.MakeWritable(setting.RepoRootPath) }) + doAPIEditRepository(rescopedCtx, &api.EditRepoOption{Private: &private})(t) + + repoURL := createSSHUrl(ctx.GitPath(), u) + + // Fill in fixture data + withAnnexCtxKeyFile(t, ctx, func() { + err = doInitRemoteAnnexRepository(t, repoURL) + }) + if err != nil { + return fmt.Errorf("Unable to initialize remote repo with git-annex fixture: %w", err) + } + return nil +} + +/* +Test that permissions are enforced on git-annex-shell commands. + + Along the way, test that uploading, downloading, and deleting all work. +*/ +func TestGitAnnexPermissions(t *testing.T) { + /* + // TODO: look into how LFS did this + if !setting.Annex.Enabled { + t.Skip() + } + */ + + // Each case below is split so that 'clone' is done as + // the repo owner, but 'copy' as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + t.Run("Public", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-public", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false)) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.False(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + + t.Run("Private", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-private", auth_model.AccessTokenScopeWriteRepository) + + // create a private repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true)) + + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + }) +} + +/* +Test that 'git annex init' works. + + precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). +*/ +func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { + _, _, err = git.NewCommand(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex init`: %w", err) + } + + // - method 0: 'git config remote.origin.annex-uuid'. + // Demonstrates that 'git annex init' successfully contacted + // the remote git-annex and was able to learn its ID number. + readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) + } + readAnnexUUID = strings.TrimSpace(readAnnexUUID) + + match := regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(readAnnexUUID) + if !match { + return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) + } + + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + if err != nil { + return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) + } + + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + match = regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(remoteAnnexUUID) + if !match { + return fmt.Errorf("'git annex init' should have been able to download the remote's uuid; but instead read '%s'", remoteAnnexUUID) + } + + if readAnnexUUID != remoteAnnexUUID { + return fmt.Errorf("'git annex init' should have read the expected annex UUID '%s', but instead got '%s'", remoteAnnexUUID, readAnnexUUID) + } + + // - method 1: 'git annex whereis'. + // Demonstrates that git-annex understands the annexed file can be found in the remote annex. + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) + } + // Note: this regex is unanchored because 'whereis' outputs multiple lines containing + // headers and 1+ remotes and we just want to find one of them. + match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) + if !match { + return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + } + + return nil +} + +func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "doAnnexInitTest()": + // "git annex copy" will notice and run "git annex init", silently. + // This shouldn't change any results, but be aware in case it does. + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was downloaded + localObjectPath, err := annexObjectPath(repoPath, "large.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. + + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was uploaded + localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +// ---- Helpers ---- + +func generateRandomFile(size int, path string) (err error) { + // Generate random file + + // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic + bufSize := 4 * 1024 + if bufSize > size { + bufSize = size + } + + buffer := make([]byte, bufSize) + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + written := 0 + for written < size { + n := size - written + if n > bufSize { + n = bufSize + } + _, err := rand.Read(buffer[:n]) + if err != nil { + return err + } + n, err = f.Write(buffer[:n]) + if err != nil { + return err + } + written += n + } + if err != nil { + return err + } + + return nil +} + +// ---- Annex-specific helpers ---- + +/* +Initialize a repo with some baseline annexed and non-annexed files. + + TODO: perhaps this generator could be replaced with a fixture (see + integrations/gitea-repositories-meta/ and models/fixtures/repository.yml). + However we reuse this template for -different- repos, so maybe not. +*/ +func doInitAnnexRepository(repoPath string) error { + // set up what files should be annexed + // in this case, all *.bin files will be annexed + // without this, git-annex's default config annexes every file larger than some number of megabytes + f, err := os.Create(path.Join(repoPath, ".gitattributes")) + if err != nil { + return err + } + defer f.Close() + + // set up git-annex to store certain filetypes via *annex* pointers + // (https://git-annex.branchable.com/internals/pointer_file/). + // but only when run via 'git add' (see git-annex-smudge(1)) + _, err = f.WriteString("* annex.largefiles=anything\n") + if err != nil { + return err + } + _, err = f.WriteString("*.bin filter=annex\n") + if err != nil { + return err + } + f.Close() + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"}) + if err != nil { + return err + } + + // 'git annex init' + err = git.NewCommand(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add a file to the annex + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + if err != nil { + return err + } + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + if err != nil { + return err + } + + return nil +} + +/* +Initialize a remote repo with some baseline annexed and non-annexed files. +*/ +func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { + repoPath := path.Join(t.TempDir(), path.Base(repoURL.Path)) + // This clone is immediately thrown away, which + // helps force the tests to be end-to-end. + defer util.RemoveAll(repoPath) + + doGitClone(repoPath, repoURL)(t) // TODO: this call is the only reason for the testing.T; can it be removed? + + err := doInitAnnexRepository(repoPath) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + return nil +} + +/* +Find the path in .git/annex/objects/ of the contents for a given annexed file. + + repoPath: the git repository to examine + file: the path (in the repo's current HEAD) of the annex pointer + + TODO: pass a parameter to allow examining non-HEAD branches +*/ +func annexObjectPath(repoPath, file string) (string, error) { + // NB: `git annex lookupkey` is more reliable, but doesn't work in bare repos. + annexKey, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "show").AddDynamicArguments("HEAD:" + file).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %w", repoPath, err) // the error from git prints the filename but not repo + } + + // There are two formats an annexed file pointer might be: + // * a symlink to .git/annex/objects/$HASHDIR/$ANNEX_KEY/$ANNEX_KEY - used by files created with 'git annex add' + // * a text file containing /annex/objects/$ANNEX_KEY - used by files for which 'git add' was configured to run git-annex-smudge + // This recovers $ANNEX_KEY from either case: + annexKey = path.Base(strings.TrimSpace(annexKey)) + + contentPath, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be annexed: %w", repoPath, file, err) + } + contentPath = strings.TrimSpace(contentPath) + + return path.Join(repoPath, contentPath), nil +} + +/* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ +func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") + defer func() { + // reset + if gitAnnexUseGitSSHExists { + os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + } + }() + + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + + withCtxKeyFile(t, ctx, callback) +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 89453296ca..8921768c1f 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -42,6 +42,28 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"IdentitiesOnly=yes\" -i \""+keyFile+"\" \"$@\""), 0o700) require.NoError(t, err) + // reset ssh wrapper afterwards + _gitSSH, gitSSHExists := os.LookupEnv("GIT_SSH") + defer func() { + if gitSSHExists { + t.Setenv("GIT_SSH", _gitSSH) + } + }() + + _gitSSHCommand, gitSSHCommandExists := os.LookupEnv("GIT_SSH_COMMAND") + defer func() { + if gitSSHCommandExists { + t.Setenv("GIT_SSH_COMMAND", _gitSSHCommand) + } + }() + + _gitSSHVariant, gitSSHVariantExists := os.LookupEnv("GIT_SSH_VARIANT") + defer func() { + if gitSSHVariantExists { + t.Setenv("GIT_SSH_VARIANT", _gitSSHVariant) + } + }() + // Setup ssh wrapper t.Setenv("GIT_SSH", path.Join(tmpDir, "ssh")) t.Setenv("GIT_SSH_COMMAND", From c821e35ce3b368f7c0d23c6c472bde93e8cf9149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 24 May 2024 12:57:45 +0200 Subject: [PATCH 05/47] Adapt patch to upstream changes Repository creation now expects an objectFormat to be specified for git. --- tests/integration/git_annex_test.go | 522 ++++++++++++++-------------- 1 file changed, 262 insertions(+), 260 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index f64b1d98df..4a2f255737 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -34,13 +34,13 @@ import ( // which is assumed to be owned by that username; if you want to target a different // repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" -func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool) (err error) { +func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool, objectFormat git.ObjectFormat) (err error) { // creating a repo counts as editing the user's profile (is done by POSTing // to /api/v1/user/repos/) -- which means it needs a User-scoped token and // both that and editing need a Repo-scoped token because they edit repositories. rescopedCtx := ctx rescopedCtx.Token = getTokenForLoggedInUser(t, ctx.Session, auth_model.AccessTokenScopeWriteUser, auth_model.AccessTokenScopeWriteRepository) - doAPICreateRepository(rescopedCtx, false)(t) + doAPICreateRepository(rescopedCtx, false, objectFormat)(t) t.Cleanup(func() { util.MakeWritable(setting.RepoRootPath) }) doAPIEditRepository(rescopedCtx, &api.EditRepoOption{Private: &private})(t) @@ -78,365 +78,367 @@ func TestGitAnnexPermissions(t *testing.T) { // 'annex copy' -- potentially leaving a security gap. onGiteaRun(t, func(t *testing.T, u *url.URL) { - t.Run("Public", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - ownerCtx := NewAPITestContext(t, "user2", "annex-public", auth_model.AccessTokenScopeWriteRepository) - - // create a public repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false)) - - // double-check it's public - repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) - require.NoError(t, err) - require.False(t, repo.IsPrivate) - - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL - remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost - - // Different sessions, so we can test different permissions. - // We leave Reponame blank because we don't actually then later add it according to each case if needed - // - // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) - readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) - outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access - - // set up collaborators - doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) - - // tests - t.Run("Owner", func(t *testing.T) { + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + t.Run("Public", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { + ownerCtx := NewAPITestContext(t, "user2", "annex-public"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false, objectFormat)) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.False(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Writer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Writer", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Reader", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Reader", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) }) }) }) - }) - t.Run("Outsider", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Outsider", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) }) }) }) - }) - t.Run("Delete", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ownerCtx)(t) - _, statErr := os.Stat(remoteRepoPath) - require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") - }) - }) - - t.Run("Private", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - ownerCtx := NewAPITestContext(t, "user2", "annex-private", auth_model.AccessTokenScopeWriteRepository) - - // create a private repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true)) - - // double-check it's private - repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) - require.NoError(t, err) - require.True(t, repo.IsPrivate) - - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL - remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost - - // Different sessions, so we can test different permissions. - // We leave Reponame blank because we don't actually then later add it according to each case if needed - // - // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) - readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) - outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access - // Note: there's also full anonymous access, which is only available for public HTTP repos; - // it should behave the same as 'outsider' but we (will) test it separately below anyway - - // set up collaborators - doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) - - // tests - t.Run("Owner", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Delete", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) - - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) - }) + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") }) }) - t.Run("Writer", func(t *testing.T) { + t.Run("Private", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { + ownerCtx := NewAPITestContext(t, "user2", "annex-private"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + + // create a private repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true, objectFormat)) + + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Reader", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Writer", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Outsider", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Reader", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) }) }) }) - }) - t.Run("Delete", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ownerCtx)(t) - _, statErr := os.Stat(remoteRepoPath) - require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) }) }) }) From ee7f8249d11a65509989022f7000ba3d5bfd1d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 30 Apr 2024 12:12:39 +0200 Subject: [PATCH 06/47] Adapt patch to upstream changes A dead code check started to complain because FileCmp was only used in tests. Moved the function to test_utils. --- modules/util/filecmp.go | 87 ----------------------------- tests/integration/git_annex_test.go | 4 +- tests/test_utils.go | 79 ++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 89 deletions(-) delete mode 100644 modules/util/filecmp.go diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go deleted file mode 100644 index 76e7705cc1..0000000000 --- a/modules/util/filecmp.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package util - -import ( - "bytes" - "io" - "os" -) - -// Decide if two files have the same contents or not. -// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. -// *Follows* symlinks. -// -// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. -// -// derived from https://stackoverflow.com/a/30038571 -// under CC-BY-SA-4.0 by several contributors -func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { - if chunkSize == 0 { - chunkSize = 4 * 1024 - } - - // shortcuts: check file metadata - stat1, err := os.Stat(file1) - if err != nil { - return false, err - } - - stat2, err := os.Stat(file2) - if err != nil { - return false, err - } - - // are inputs are literally the same file? - if os.SameFile(stat1, stat2) { - return true, nil - } - - // do inputs at least have the same size? - if stat1.Size() != stat2.Size() { - return false, nil - } - - // long way: compare contents - f1, err := os.Open(file1) - if err != nil { - return false, err - } - defer f1.Close() - - f2, err := os.Open(file2) - if err != nil { - return false, err - } - defer f2.Close() - - b1 := make([]byte, chunkSize) - b2 := make([]byte, chunkSize) - for { - n1, err1 := io.ReadFull(f1, b1) - n2, err2 := io.ReadFull(f2, b2) - - // https://pkg.go.dev/io#Reader - // > Callers should always process the n > 0 bytes returned - // > before considering the error err. Doing so correctly - // > handles I/O errors that happen after reading some bytes - // > and also both of the allowed EOF behaviors. - - if !bytes.Equal(b1[:n1], b2[:n2]) { - return false, nil - } - - if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { - return true, nil - } - - // some other error, like a dropped network connection or a bad transfer - if err1 != nil { - return false, err1 - } - if err2 != nil { - return false, err2 - } - } -} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 4a2f255737..0791e287ee 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -522,7 +522,7 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) if err != nil { return err } @@ -575,7 +575,7 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) if err != nil { return err } diff --git a/tests/test_utils.go b/tests/test_utils.go index e6aa9c7963..ed795facf8 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -5,9 +5,11 @@ package tests import ( + "bytes" "context" "database/sql" "fmt" + "io" "os" "path" "path/filepath" @@ -483,3 +485,80 @@ func CreateDeclarativeRepo(t *testing.T, owner *user_model.User, name string, en return CreateDeclarativeRepoWithOptions(t, owner, opts) } + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file2) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} From aa1b739eada11deadda74dacd5c6ba4c6941ca8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 12 Aug 2024 11:34:45 +0200 Subject: [PATCH 07/47] Adapt patch to upstream changes --- tests/integration/git_annex_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 0791e287ee..8ebd834da0 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -752,11 +752,11 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { defer func() { // reset if gitAnnexUseGitSSHExists { - os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + t.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) } }() - os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + t.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set withCtxKeyFile(t, ctx, callback) } From ed712c00ba8dc81de89cd86b546023bfc6e42b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 18 Apr 2024 17:25:32 +0200 Subject: [PATCH 08/47] Install git-annex in the testing workflow --- .forgejo/workflows/testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 9ba8c47a01..ec23afe1f2 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -196,7 +196,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-lfs + packages: git git-annex git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-mysql-migration test-mysql' @@ -232,7 +232,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-lfs + packages: git git-annex git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-pgsql-migration test-pgsql' @@ -253,7 +253,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-lfs + packages: git git-annex git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-sqlite-migration test-sqlite' From 9482ff6dd123666998ea28d4ce1e838d58be6097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 25 Apr 2024 10:21:08 +0200 Subject: [PATCH 09/47] Fix failing tests Multiple tests that worked fine on v1.20.4-1 started to fail after the rebase onto v1.20.5-1. These tests are: - TestGitAnnexPermissions/Private/Owner/HTTP/Init - TestGitAnnexPermissions/Private/Owner/HTTP/Download - TestGitAnnexPermissions/Private/Writer/HTTP/Init - TestGitAnnexPermissions/Private/Writer/HTTP/Download - TestGitAnnexPermissions/Private/Reader/HTTP/Init - TestGitAnnexPermissions/Private/Reader/HTTP/Download What these tests have in common is that they all operate on a private repository via http with authentication. They broke at some point between v1.20.4-1 and v1.20.5-1, so I did a bisect between these two points running the offending tests. This brought me to the conclusion that ee48c0d5ea8f148c7f4a792b590d93eb51cbf67d introduced the issue. The thing is, this commit does not change any code, it only changes the test environment. Among other things that didn't look as suspicious, it changes the container image from a bespoke test_env image based on debian bullseye to a node image based on debian bookworm. Obviously, this means that there are many version differences between the two. The first one I looked at was git. The previous bullseye image used a manually installed git version 2.40.0, while the bookworm image has 2.39.2 installed. Updating git in the new image did not fix the issue, however. The next thing I looked at was the git-annex version. Bullseye had 8.20210223 installed and worked, while bookworm used 10.20230126 when the tests broke. So I tried my luck upgrading to a more recent version via neurodebian (10.20240227-1~ndall+1). This still worked fine on bullseye and now also works fine on bookworm. I have no idea why this specific version of git-annex broke the tests, but at least there was a commit to pinpoint this to, which isn't always the case with docker images silently changing beneath you... Below are the versions as they are reported by git and git-annex: bullseye (works): git version 2.30.2 git-annex version: 8.20210223 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 bullseye + git-annex from neurodebian (works): git version 2.30.2 git-annex version: 10.20240227-1~ndall+1 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 9 10 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 bookworm (fails): git version 2.39.2 git-annex version: 10.20230126 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 9 10 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 bookworm + git-annex from neurodebian (works): git version 2.39.2 git-annex version: 10.20240227-1~ndall+1 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 9 10 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 --- .forgejo/workflows-composite/apt-install-from/action.yaml | 3 +++ .forgejo/workflows/testing.yml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows-composite/apt-install-from/action.yaml b/.forgejo/workflows-composite/apt-install-from/action.yaml index 615e7cb184..ab55883a11 100644 --- a/.forgejo/workflows-composite/apt-install-from/action.yaml +++ b/.forgejo/workflows-composite/apt-install-from/action.yaml @@ -13,6 +13,8 @@ runs: run: | export DEBIAN_FRONTEND=noninteractive echo "deb http://deb.debian.org/debian/ ${RELEASE} main" > "/etc/apt/sources.list.d/${RELEASE}.list" + wget -O- http://neuro.debian.net/lists/bookworm.de-fzj.libre | tee /etc/apt/sources.list.d/neurodebian.sources.list + apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 env: RELEASE: ${{inputs.release}} - name: install packages @@ -24,6 +26,7 @@ runs: - name: remove temporary package list to prevent using it in other steps run: | rm "/etc/apt/sources.list.d/${RELEASE}.list" + rm "/etc/apt/sources.list.d/neurodebian.sources.list" apt-get update -qq env: RELEASE: ${{inputs.release}} diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index ec23afe1f2..7e3a951872 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -196,7 +196,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-annex git-lfs + packages: git git-annex-standalone git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-mysql-migration test-mysql' @@ -232,7 +232,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-annex git-lfs + packages: git git-annex-standalone git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-pgsql-migration test-pgsql' @@ -253,7 +253,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-annex git-lfs + packages: git git-annex-standalone git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-sqlite-migration test-sqlite' From 90d53d48a5f8167bbcadedfecbea2071063adb94 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 19 Sep 2022 17:22:42 -0400 Subject: [PATCH 10/47] git-annex: add configuration setting [annex].ENABLED Fixes https://github.com/neuropoly/gitea/issues/8 Co-authored-by: Mathieu Guay-Paquet --- cmd/serv.go | 3 +-- cmd/web.go | 4 ++++ custom/conf/app.example.ini | 9 +++++++++ modules/setting/annex.go | 20 ++++++++++++++++++++ modules/setting/setting.go | 1 + routers/private/serv.go | 2 +- tests/integration/git_annex_test.go | 9 +++------ tests/mysql.ini.tmpl | 3 +++ tests/pgsql.ini.tmpl | 3 +++ tests/sqlite.ini.tmpl | 3 +++ 10 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 modules/setting/annex.go diff --git a/cmd/serv.go b/cmd/serv.go index 59379a91b4..62e03dfd3b 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -215,8 +215,7 @@ func runServ(c *cli.Context) error { } if verb == gitAnnexShellVerb { - // if !setting.Annex.Enabled { // TODO: https://github.com/neuropoly/gitea/issues/8 - if false { + if !setting.Annex.Enabled { return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") } diff --git a/cmd/web.go b/cmd/web.go index 3e7fdee4bf..ecc53698e3 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -322,6 +322,10 @@ func listen(m http.Handler, handleRedirector bool) error { log.Info("LFS server enabled") } + if setting.Annex.Enabled { + log.Info("git-annex enabled") + } + var err error switch setting.Protocol { case setting.HTTP: diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index f8fa95bbab..789178d9be 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2678,6 +2678,15 @@ LEVEL = Info ;; Limit the number of concurrent upload/download operations within a batch ;BATCH_OPERATION_CONCURRENCY = 8 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[annex] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Whether git-annex is enabled; defaults to false +;ENABLED = false + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; settings for packages, will override storage setting diff --git a/modules/setting/annex.go b/modules/setting/annex.go new file mode 100644 index 0000000000..b8616ba87d --- /dev/null +++ b/modules/setting/annex.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "forgejo.org/modules/log" +) + +// Annex represents the configuration for git-annex +var Annex = struct { + Enabled bool `ini:"ENABLED"` +}{} + +func loadAnnexFrom(rootCfg ConfigProvider) { + sec := rootCfg.Section("annex") + if err := sec.MapTo(&Annex); err != nil { + log.Fatal("Failed to map Annex settings: %v", err) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index eb7b9e9373..7c40f6057e 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -148,6 +148,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error { loadCamoFrom(cfg) loadI18nFrom(cfg) loadGitFrom(cfg) + loadAnnexFrom(cfg) loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadQuotaFrom(cfg) diff --git a/routers/private/serv.go b/routers/private/serv.go index 82953b5af9..5d3ab0ed78 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -289,7 +289,7 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey - ( /*setting.Annex.Enabled && */ len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode + (setting.Annex.Enabled && len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { results.UserMode = deployKey.Mode diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 8ebd834da0..27d7267fb6 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -62,12 +62,9 @@ Test that permissions are enforced on git-annex-shell commands. Along the way, test that uploading, downloading, and deleting all work. */ func TestGitAnnexPermissions(t *testing.T) { - /* - // TODO: look into how LFS did this - if !setting.Annex.Enabled { - t.Skip() - } - */ + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } // Each case below is split so that 'clone' is done as // the repo owner, but 'copy' as the user under test. diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index e15e79952b..b8320265ab 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -97,6 +97,9 @@ DISABLE_QUERY_AUTH_TOKEN = true [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index 340531fb38..781508a648 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -122,6 +122,9 @@ MINIO_LOCATION = us-east-1 MINIO_USE_SSL = false MINIO_CHECKSUM_ALGORITHM = md5 +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index 277916a539..231c0d19c2 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -102,6 +102,9 @@ JWT_SECRET = KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true From 3d9d84ad70fc4000615dfbbc7c4f4d61e13d3501 Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 20 Sep 2022 16:17:56 -0400 Subject: [PATCH 11/47] git-annex: support downloading over HTTP This makes HTTP symmetric with SSH clone URLs. This gives us the fancy feature of _anonymous_ downloads, so people can access datasets without having to set up an account or manage ssh keys. Previously, to access "open access" data shared this way, users would need to: 1. Create an account on gitea.example.com 2. Create ssh keys 3. Upload ssh keys (and make sure to find and upload the correct file) 4. `git clone git@gitea.example.com:user/dataset.git` 5. `cd dataset` 6. `git annex get` This cuts that down to just the last three steps: 1. `git clone https://gitea.example.com/user/dataset.git` 2. `cd dataset` 3. `git annex get` This is significantly simpler for downstream users, especially for those unfamiliar with the command line. Unfortunately there's no uploading. While git-annex supports uploading over HTTP to S3 and some other special remotes, it seems to fail on a _plain_ HTTP remote. See https://github.com/neuropoly/gitea/issues/7 and https://git-annex.branchable.com/forum/HTTP_uploads/#comment-ce28adc128fdefe4c4c49628174d9b92. This is not a major loss since no one wants uploading to be anonymous anyway. To support private repos, I had to hunt down and patch a secret extra security corner that Gitea only applies to HTTP for some reason (services/auth/basic.go). This was guided by https://git-annex.branchable.com/tips/setup_a_public_repository_on_a_web_site/ Fixes https://github.com/neuropoly/gitea/issues/3 Co-authored-by: Mathieu Guay-Paquet --- modules/git/command.go | 3 +- routers/web/repo/githttp.go | 31 ++ routers/web/web.go | 13 + services/auth/auth.go | 11 + services/auth/basic.go | 4 +- tests/integration/git_annex_test.go | 360 +++++++++++++++++- .../git_helper_for_declarative_test.go | 7 + 7 files changed, 412 insertions(+), 17 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index bf1d624dbf..fd29ac36e9 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -447,12 +447,13 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests +// It also re-enables git-credential(1), which is used to test git-annex's HTTP support func AllowLFSFiltersArgs() TrustedCmdArgs { // Now here we should explicitly allow lfs filters to run filteredLFSGlobalArgs := make(TrustedCmdArgs, len(globalCommandArgs)) j := 0 for _, arg := range globalCommandArgs { - if strings.Contains(string(arg), "lfs") { + if strings.Contains(string(arg), "lfs") || strings.Contains(string(arg), "credential") { j-- } else { filteredLFSGlobalArgs[j] = arg diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index 650b1d88f4..019befc303 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -597,3 +597,34 @@ func GetIdxFile(ctx *context.Context) { h.sendFile(ctx, "application/x-git-packed-objects-toc", "objects/pack/pack-"+ctx.Params("file")+".idx") } } + +// GetAnnexObject implements git-annex dumb HTTP +func GetAnnexObject(ctx *context.Context) { + h := httpBase(ctx) + if h != nil { + // git-annex objects are stored in .git/annex/objects/{hash1}/{hash2}/{key}/{key} + // where key is a string containing the size and (usually SHA256) checksum of the file, + // and hash1+hash2 are the first few bits of the md5sum of key itself. + // ({hash1}/{hash2}/ is just there to avoid putting too many files in one directory) + // ref: https://git-annex.branchable.com/internals/hashing/ + + // keyDir should = key, but we don't enforce that + object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + + // Sanitize the input against directory traversals. + // + // This works because at the filesystem root, "/.." = "/"; + // So if a path starts rooted ("/"), path.Clean(), which + // path.Join() calls internally, removes all '..' prefixes. + // After, this unroots the path unconditionally ([1:]), which + // works because we know the input is never supposed to be rooted. + // + // The router code probably also disallows "..", so this + // should be redundant, but it's defensive to keep it + // whenever touching filesystem paths with user input. + object = path.Join("/", object)[1:] + + h.setHeaderCacheForever() + h.sendFile("application/octet-stream", "annex/objects/"+object) + } +} diff --git a/routers/web/web.go b/routers/web/web.go index 303167a6b9..78ab60545b 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -350,6 +350,13 @@ func registerRoutes(m *web.Route) { } } + annexEnabled := func(ctx *context.Context) { + if !setting.Annex.Enabled { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -1632,6 +1639,12 @@ func registerRoutes(m *web.Route) { }) }, ignSignInAndCsrf, lfsServerEnabled) + m.Group("", func() { + // for git-annex + m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + gitHTTPRouters(m) }) }) diff --git a/services/auth/auth.go b/services/auth/auth.go index 85c9296ced..9d72a88c1f 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -61,6 +61,17 @@ func isArchivePath(req *http.Request) bool { return archivePathRe.MatchString(req.URL.Path) } +var annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) + +func isAnnexPath(req *http.Request) bool { + if setting.Annex.Enabled { + // "/config" is git's config, not specifically git-annex's; but the only current + // user of it is when git-annex downloads the annex.uuid during 'git annex init'. + return strings.HasSuffix(req.URL.Path, "/config") || annexPathRe.MatchString(req.URL.Path) + } + return false +} + // handleSignIn clears existing session variables and stores new ones for the specified user object func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore, user *user_model.User) { // We need to regenerate the session... diff --git a/services/auth/basic.go b/services/auth/basic.go index f259ad5f69..7adcd8914a 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -43,8 +43,8 @@ func (b *Basic) Name() string { // name/token on successful validation. // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { - // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) { + // Basic authentication should only fire on API, Download or on Git, LFSPaths or Git-Annex paths + if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) && !isAnnexPath(req) { return nil, nil } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 27d7267fb6..cea52d813f 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -59,7 +59,8 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, /* Test that permissions are enforced on git-annex-shell commands. - Along the way, test that uploading, downloading, and deleting all work. + Along the way, this also tests that uploading, downloading, and deleting all work, + so we haven't written separate tests for those. */ func TestGitAnnexPermissions(t *testing.T) { if !setting.Annex.Enabled { @@ -75,6 +76,16 @@ func TestGitAnnexPermissions(t *testing.T) { // 'annex copy' -- potentially leaving a security gap. onGiteaRun(t, func(t *testing.T, u *url.URL) { + // Tell git-annex to allow http://127.0.0.1, http://localhost and http://::1. Without + // this, all `git annex` commands will silently fail when run against http:// remotes + // without explaining what's wrong. + // + // Note: onGiteaRun() sets up an alternate HOME so this actually edits + // tests/integration/gitea-integration-*/data/home/.gitconfig and + // if you're debugging you need to remember to match that. + _, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("annex.security.allowed-ip-addresses", "all").RunStdString(&git.RunOpts{}) + require.NoError(t, err) + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { t.Run("Public", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -89,8 +100,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.False(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -112,6 +121,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -139,6 +150,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -147,6 +183,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -174,6 +212,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -182,6 +245,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -209,6 +274,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -217,6 +307,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -244,6 +336,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -269,8 +416,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.True(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -294,6 +439,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -321,6 +468,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -329,6 +501,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -356,6 +530,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -364,6 +563,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -391,6 +592,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -399,6 +625,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -426,6 +654,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -447,7 +730,7 @@ Test that 'git annex init' works. precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). */ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { - _, _, err = git.NewCommand(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex init`: %w", err) } @@ -455,7 +738,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { // - method 0: 'git config remote.origin.annex-uuid'. // Demonstrates that 'git annex init' successfully contacted // the remote git-annex and was able to learn its ID number. - readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + readAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) } @@ -466,7 +749,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) } - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + remoteAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) if err != nil { return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) } @@ -483,7 +766,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { // - method 1: 'git annex whereis'. // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) } @@ -502,7 +785,7 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { // "git annex copy" will notice and run "git annex init", silently. // This shouldn't change any results, but be aware in case it does. - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -550,12 +833,12 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -667,7 +950,7 @@ func doInitAnnexRepository(repoPath string) error { } // 'git annex init' - err = git.NewCommand(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -705,7 +988,7 @@ func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -757,3 +1040,52 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { withCtxKeyFile(t, ctx, callback) } + +/* +Like withKeyFile(), but sets HTTP credentials instead of SSH credentials. + + It does this by temporarily arranging through `git config --global` + to use git-credential-store(1) with the password written to a tempfile. + + This is the only reliable way to pass HTTP credentials non-interactively + to git-annex. See https://git-annex.branchable.com/bugs/http_remotes_ignore_annex.web-options_--netrc/#comment-b5a299e9826b322f2d85c96d4929a430 + for joeyh's proclamation on the subject. + + This **is only effective** when used around git.NewCommandContextNoGlobals() calls. + git.NewCommand() disables credential.helper as a precaution (see modules/git/git.go). + + In contrast, the tests in git_test.go put the password in the remote's URL like + `git config remote.origin.url http://user2:password@localhost:3003/user2/repo-name.git`, + writing the password in repoPath+"/.git/config". That would be equally good, except + that git-annex ignores it! +*/ +func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, callback func()) { + credentialedURL := *u + credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + + creds := path.Join(t.TempDir(), "creds") + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) + + originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) + if err != nil && !err.IsExitCode(1) { + // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) + // but catch all others + require.NoError(t, err) + } + hasOriginalCredentialHelper := (err == nil) + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper", fmt.Sprintf("store --file=%s", creds)).RunStdString(&git.RunOpts{}) + require.NoError(t, err) + + defer (func() { + // reset + if hasOriginalCredentialHelper { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("credential.helper").AddDynamicArguments(originalCredentialHelper).RunStdString(&git.RunOpts{}) + } else { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddOptionValues("--unset").AddArguments("credential.helper").RunStdString(&git.RunOpts{}) + } + require.NoError(t, err) + })() + + callback() +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 8921768c1f..7b00ebd3a1 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -73,6 +73,13 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } +func createHTTPUrl(gitPath string, u *url.URL) *url.URL { + // this assumes u contains the HTTP base URL that Gitea is running on + u2 := *u + u2.Path = gitPath + return &u2 +} + func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh" From bb96dc35e9bf3120182a3fa40cf8541f062d2070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 1 Mar 2024 14:23:33 +0100 Subject: [PATCH 12/47] Replace m.GetOptions with m.Methods This applies the same changes that were done in 265cd70bdb152291a13e520cff1da70b8c029432 to the git-annex specific routes as well. --- routers/web/web.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/web.go b/routers/web/web.go index 78ab60545b..eb17259864 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1641,8 +1641,8 @@ func registerRoutes(m *web.Route) { m.Group("", func() { // for git-annex - m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` - m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + m.Methods("GET,OPTIONS", "/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.Methods("GET,OPTIONS", "/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) gitHTTPRouters(m) From e4c5a6449946acb939fcb22662e749b9b7533f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 29 Apr 2024 15:52:47 +0200 Subject: [PATCH 13/47] Fix exit code check for git command The err.IsExitCode method was changed to a function IsErrorExitCode taking err as its first argument in 1e7a6483b8322ad5e1183545a6283f137a0546ac. --- tests/integration/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index cea52d813f..3eff7c7062 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -1067,7 +1067,7 @@ func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, call require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) - if err != nil && !err.IsExitCode(1) { + if err != nil && !git.IsErrorExitCode(err, 1) { // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) // but catch all others require.NoError(t, err) From 5288b230d2020a215a64b07dae76faf005b82283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 29 Apr 2024 17:44:58 +0200 Subject: [PATCH 14/47] Adapt patch to upstream changes Usage of `path` was replaced by `path/filepath` in upstream forgejo, and it made sense to use that as well where `path` was previously used. The `setHeaderCacheForever` function and the `sendFile` method had their signature changed. --- routers/web/repo/githttp.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index 019befc303..3c95f4bd4c 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -609,7 +609,7 @@ func GetAnnexObject(ctx *context.Context) { // ref: https://git-annex.branchable.com/internals/hashing/ // keyDir should = key, but we don't enforce that - object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + object := filepath.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) // Sanitize the input against directory traversals. // @@ -622,9 +622,9 @@ func GetAnnexObject(ctx *context.Context) { // The router code probably also disallows "..", so this // should be redundant, but it's defensive to keep it // whenever touching filesystem paths with user input. - object = path.Join("/", object)[1:] + object = filepath.Join(string(filepath.Separator), object)[1:] - h.setHeaderCacheForever() - h.sendFile("application/octet-stream", "annex/objects/"+object) + setHeaderCacheForever(ctx) + h.sendFile(ctx, "application/octet-stream", "annex/objects/"+object) } } From 75afe98f5c6344da56b494772d75290bc90c1d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 30 Apr 2024 11:25:16 +0200 Subject: [PATCH 15/47] Adapt patch to upstream changes The "context_service" import was changed to use the default name of just "context". The patch set had to be adapted for that. --- routers/web/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/web/web.go b/routers/web/web.go index eb17259864..91400d5d17 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1643,7 +1643,7 @@ func registerRoutes(m *web.Route) { // for git-annex m.Methods("GET,OPTIONS", "/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` m.Methods("GET,OPTIONS", "/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) - }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + }, ignSignInAndCsrf, annexEnabled, context.UserAssignmentWeb()) gitHTTPRouters(m) }) From 65ed342bf973a33cad9125647c1afd007cc2f67f Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 00:28:55 -0500 Subject: [PATCH 16/47] git-annex: create modules/annex This moves the `annexObjectPath()` helper out of the tests and into a dedicated sub-package as `annex.ContentLocation()`, and expands it with `.Pointer()` (which validates using `git annex examinekey`), `.IsAnnexed()` and `.Content()` to make it a more useful module. The tests retain their own wrapper version of `ContentLocation()` because I tried to follow close to the API modules/lfs uses, which in terms of abstract `git.Blob` and `git.TreeEntry` objects, not in terms of `repoPath string`s which are more convenient for the tests. --- modules/annex/annex.go | 154 ++++++++++++++++++++++++++++ modules/git/blob.go | 4 + tests/integration/git_annex_test.go | 41 ++++---- 3 files changed, 181 insertions(+), 18 deletions(-) create mode 100644 modules/annex/annex.go diff --git a/modules/annex/annex.go b/modules/annex/annex.go new file mode 100644 index 0000000000..96b604693e --- /dev/null +++ b/modules/annex/annex.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Unlike modules/lfs, which operates mainly on git.Blobs, this operates on git.TreeEntrys. +// The motivation for this is that TreeEntrys have an easy pointer to the on-disk repo path, +// while blobs do not (in fact, if building with TAGS=gogit, blobs might exist only in a mock +// filesystem, living only in process RAM). We must have the on-disk path to do anything +// useful with git-annex because all of its interesting data is on-disk under .git/annex/. + +package annex + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "forgejo.org/modules/git" + "forgejo.org/modules/setting" + "forgejo.org/modules/util" +) + +const ( + // > The maximum size of a pointer file is 32 kb. + // - https://git-annex.branchable.com/internals/pointer_file/ + // It's unclear if that's kilobytes or kibibytes; assuming kibibytes: + blobSizeCutoff = 32 * 1024 +) + +// ErrInvalidPointer occurs if the pointer's value doesn't parse +var ErrInvalidPointer = errors.New("Not a git-annex pointer") + +// Gets the content of the blob as raw text, up to n bytes. +// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit) +func getBlobContent(b *git.Blob, n int) (string, error) { + dataRc, err := b.DataAsync() + if err != nil { + return "", err + } + defer dataRc.Close() + buf := make([]byte, n) + n, _ = util.ReadAtMost(dataRc, buf) + buf = buf[:n] + return string(buf), nil +} + +func Pointer(blob *git.Blob) (string, error) { + // git-annex doesn't seem fully spec what its pointer are, but + // the fullest description is here: + // https://git-annex.branchable.com/internals/pointer_file/ + + // a pointer can be: + // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY' + // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY' + // + // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because + // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one. + + if blob.Size() > blobSizeCutoff { + // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file. + // https://git-annex.branchable.com/internals/pointer_file/ + + // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too. + return "", ErrInvalidPointer + } + + pointer, err := getBlobContent(blob, blobSizeCutoff) + if err != nil { + return "", fmt.Errorf("error reading %s: %w", blob.Name(), err) + } + + // the spec says a pointer file can contain multiple lines each with a pointer in them + // but that makes no sense to me, so I'm just ignoring all but the first + lines := strings.Split(pointer, "\n") + if len(lines) < 1 { + return "", ErrInvalidPointer + } + pointer = lines[0] + + // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it + if !strings.Contains(pointer, "/annex/") { + return "", ErrInvalidPointer + } + + // extract $KEY + pointer = path.Base(strings.TrimSpace(pointer)) + + // ask git-annex's opinion on $KEY + // XXX: this is probably a bit slow, especially if this operation gets run often + // and examinekey is not that strict: + // - it doesn't enforce that the "BACKEND" tag is one it knows, + // - it doesn't enforce that the fields and their format fit the "BACKEND" tag + // so maybe this is a wasteful step + _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + // TODO: make ErrInvalidPointer into a type capable of wrapping err + if strings.TrimSpace(examineStderr) == "git-annex: bad key" { + return "", ErrInvalidPointer + } + return "", err + } + + return pointer, nil +} + +// return the absolute path of the content pointed to by the annex pointer stored in the git object +// errors if the content is not found in this repo +func ContentLocation(blob *git.Blob) (string, error) { + pointer, err := Pointer(blob) + if err != nil { + return "", err + } + + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err) + } + contentLocation = strings.TrimSpace(contentLocation) + contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals + contentLocation = path.Join(blob.Repo().Path, contentLocation) + + return contentLocation, nil +} + +// returns a stream open to the annex content +func Content(blob *git.Blob) (*os.File, error) { + contentLocation, err := ContentLocation(blob) + if err != nil { + return nil, err + } + + return os.Open(contentLocation) +} + +// whether the object appears to be a valid annex pointer +// does *not* verify if the content is actually in this repo; +// for that, use ContentLocation() +func IsAnnexed(blob *git.Blob) (bool, error) { + if !setting.Annex.Enabled { + return false, nil + } + + // Pointer() is written to only return well-formed pointers + // so the test is just to see if it errors + _, err := Pointer(blob) + if err != nil { + if errors.Is(err, ErrInvalidPointer) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/modules/git/blob.go b/modules/git/blob.go index 3fda358938..8f912189ed 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -126,6 +126,10 @@ func (b *blobReader) Close() error { return nil } +func (b *Blob) Repo() *Repository { + return b.repo +} + // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { return b.name diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 3eff7c7062..a583e41df2 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -19,6 +19,7 @@ import ( "forgejo.org/models/db" "forgejo.org/models/perm" repo_model "forgejo.org/models/repo" + "forgejo.org/modules/annex" "forgejo.org/modules/git" "forgejo.org/modules/setting" api "forgejo.org/modules/structs" @@ -791,13 +792,13 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { } // verify the file was downloaded - localObjectPath, err := annexObjectPath(repoPath, "large.bin") + localObjectPath, err := contentLocation(repoPath, "large.bin") if err != nil { return err } // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file - remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin") if err != nil { return err } @@ -844,13 +845,13 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { } // verify the file was uploaded - localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + localObjectPath, err := contentLocation(repoPath, "contribution.bin") if err != nil { return err } // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file - remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + remoteObjectPath, err := contentLocation(remoteRepoPath, "contribution.bin") if err != nil { return err } @@ -1004,26 +1005,30 @@ Find the path in .git/annex/objects/ of the contents for a given annexed file. TODO: pass a parameter to allow examining non-HEAD branches */ -func annexObjectPath(repoPath, file string) (string, error) { - // NB: `git annex lookupkey` is more reliable, but doesn't work in bare repos. - annexKey, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "show").AddDynamicArguments("HEAD:" + file).RunStdString(&git.RunOpts{Dir: repoPath}) +func contentLocation(repoPath, file string) (path string, err error) { + path = "" + + repo, err := git.OpenRepository(git.DefaultContext, repoPath) if err != nil { - return "", fmt.Errorf("in %s: %w", repoPath, err) // the error from git prints the filename but not repo + return path, nil } - // There are two formats an annexed file pointer might be: - // * a symlink to .git/annex/objects/$HASHDIR/$ANNEX_KEY/$ANNEX_KEY - used by files created with 'git annex add' - // * a text file containing /annex/objects/$ANNEX_KEY - used by files for which 'git add' was configured to run git-annex-smudge - // This recovers $ANNEX_KEY from either case: - annexKey = path.Base(strings.TrimSpace(annexKey)) - - contentPath, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be annexed: %w", repoPath, file, err) + return path, nil } - contentPath = strings.TrimSpace(contentPath) - return path.Join(repoPath, contentPath), nil + commit, err := repo.GetCommit(commitID) + if err != nil { + return path, nil + } + + treeEntry, err := commit.GetTreeEntryByPath(file) + if err != nil { + return path, nil + } + + return annex.ContentLocation(treeEntry.Blob()) } /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ From 98e5f6612259bc7c90954948659ee9d6410e29f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 14 May 2024 11:48:43 +0200 Subject: [PATCH 17/47] Adapt patch to upstream changes The git repository must be closed after using it. Without this change some tests started to fail due to the lingering repository running into a timeout. --- tests/integration/git_annex_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index a583e41df2..4deb6ba7e9 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -1012,6 +1012,7 @@ func contentLocation(repoPath, file string) (path string, err error) { if err != nil { return path, nil } + defer repo.Close() commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags if err != nil { From f33d0976c66e8eb264f0a105eb0305762cc6b30c Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 00:40:06 -0500 Subject: [PATCH 18/47] git-annex: make /media/ download annexed content Previously, Gitea's LFS support allowed direct-downloads of LFS content, via http://$HOSTNAME:$PORT/$USER/$REPO/media/branch/$BRANCH/$FILE Expand that grace to git-annex too. Now /media should provide the relevant *content* from the .git/annex/objects/ folder. This adds tests too. And expands the tests to try symlink-based annexing, since /media implicitly supports both that and pointer-file-based annexing. --- routers/web/repo/download.go | 21 ++++ tests/integration/git_annex_test.go | 146 ++++++++++++++++++++++++---- 2 files changed, 148 insertions(+), 19 deletions(-) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index fc82ece4cb..50ad85735b 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -8,6 +8,7 @@ import ( "time" git_model "forgejo.org/models/git" + "forgejo.org/modules/annex" "forgejo.org/modules/git" "forgejo.org/modules/httpcache" "forgejo.org/modules/lfs" @@ -78,6 +79,26 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim } closed = true + // check for git-annex files + // (this code is weirdly redundant because I'm trying not to delete any lines in order to make merges easier) + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + ctx.ServerError("annex.IsAnnexed", err) + return err + } + if isAnnexed { + content, err := annex.Content(blob) + if err != nil { + // XXX are there any other possible failure cases here? + // there are, there could be unrelated io errors; those should be ctx.ServerError()s + ctx.NotFound("annex.Content", err) + return err + } + defer content.Close() + common.ServeContentByReadSeeker(ctx.Base, ctx.Repo.TreePath, lastModified, content) + return nil + } + return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified) } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 4deb6ba7e9..df933add3a 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -7,7 +7,9 @@ package integration import ( "errors" "fmt" + "io" "math/rand" + "net/http" "net/url" "os" "path" @@ -57,6 +59,63 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, return nil } +func TestGitAnnexMedia(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + // the filenames here correspond to specific cases defined in doInitAnnexRepository() + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.bin") + }) + }) +} + +func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { + // Make sure that downloading via /media on the website recognizes it should give the annexed content + + // TODO: + // - [ ] roll this into TestGitAnnexPermissions to ensure that permission enforcement works correctly even on /media? + + session := loginUser(t, ctx.Username) // logs in to the http:// site/API, storing a cookie; + // this is a different auth method than the git+ssh:// or git+http:// protocols TestGitAnnexPermissions uses! + + // compute server-side path of the annexed file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + remoteObjectPath, err := contentLocation(remoteRepoPath, file) + require.NoError(t, err) + + // download annexed file + localObjectPath := path.Join(t.TempDir(), file) + fd, err := os.OpenFile(localObjectPath, os.O_CREATE|os.O_WRONLY, 0o777) + defer fd.Close() + require.NoError(t, err) + + mediaLink := path.Join("/", ctx.Username, ctx.Reponame, "/media/branch/master", file) + req := NewRequest(t, "GET", mediaLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + _, err = io.Copy(fd, resp.Body) + require.NoError(t, err) + fd.Close() + + // verify the download + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + require.NoError(t, err) + require.True(t, match, "Annexed files should be the same") +} + /* Test that permissions are enforced on git-annex-shell commands. @@ -766,16 +825,16 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { } // - method 1: 'git annex whereis'. - // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + // Demonstrates that git-annex understands annexed files can be found in the remote annex. + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "annexed.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { - return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) + return fmt.Errorf("Couldn't `git annex whereis`: %w", err) } // Note: this regex is unanchored because 'whereis' outputs multiple lines containing // headers and 1+ remotes and we just want to find one of them. match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) if !match { - return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + return errors.New("'git annex whereis' should report files are known to be in [origin]") } return nil @@ -791,27 +850,56 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } - // verify the file was downloaded - localObjectPath, err := contentLocation(repoPath, "large.bin") - if err != nil { - return err - } - // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file + // verify the files downloaded - remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin") + cmp := func(filename string) error { + localObjectPath, err := contentLocation(repoPath, filename) + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, filename) // or, just compare against the checked-out file + + remoteObjectPath, err := contentLocation(remoteRepoPath, filename) + if err != nil { + return err + } + + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil + } + + // this is the annex-symlink file + stat, err := os.Lstat(path.Join(repoPath, "annexed.tiff")) if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + if !((stat.Mode() & os.ModeSymlink) != 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.tiff should be a symlink") + } + if err = cmp("annexed.tiff"); err != nil { return err } - match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) + // this is the annex-pointer file + stat, err = os.Lstat(path.Join(repoPath, "annexed.bin")) if err != nil { - return err + return fmt.Errorf("Lstat: %w", err) } - if !match { - return errors.New("Annexed files should be the same") + if !((stat.Mode() & os.ModeSymlink) == 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.bin should not be a symlink") } + err = cmp("annexed.bin") - return nil + return err } func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { @@ -956,16 +1044,36 @@ func doInitAnnexRepository(repoPath string) error { return err } - // add a file to the annex - err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + // add files to the annex, stored via annex symlinks + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.tiff")) if err != nil { return err } + + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add files to the annex, stored via git-annex-smudge + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.bin")) + if err != nil { + return err + } + + if err != nil { + return err + } + err = git.AddChanges(repoPath, false, ".") if err != nil { return err } - err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + + // save everything + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex files"}) if err != nil { return err } From 3a689a351461689d453cc7e8e4597e17c3536a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 24 May 2024 13:21:20 +0200 Subject: [PATCH 19/47] Adapt patch to upstream changes Test with different objectFormats. --- tests/integration/git_annex_test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index df933add3a..d9f056f056 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -65,19 +65,21 @@ func TestGitAnnexMedia(t *testing.T) { } onGiteaRun(t, func(t *testing.T, u *url.URL) { - ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository) + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-media-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) - // create a public repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) - // the filenames here correspond to specific cases defined in doInitAnnexRepository() - t.Run("AnnexSymlink", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doAnnexMediaTest(t, ctx, "annexed.tiff") - }) - t.Run("AnnexPointer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doAnnexMediaTest(t, ctx, "annexed.bin") + // the filenames here correspond to specific cases defined in doInitAnnexRepository() + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.bin") + }) }) }) } From 276114f289e80d0bf05b2afffd1c723f4b3ee1d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 14 May 2024 14:34:44 +0200 Subject: [PATCH 20/47] Adapt patch to upstream changes Use tests.FileCmp instead of util.FileCmp. --- tests/integration/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index d9f056f056..3e54ace2f2 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -113,7 +113,7 @@ func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { fd.Close() // verify the download - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) require.NoError(t, err) require.True(t, match, "Annexed files should be the same") } From 3b7db37075270ac277e5e98090c45fe269af92ac Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 02:13:46 -0500 Subject: [PATCH 21/47] git-annex: views for annex files This updates the repo index/file view endpoints so annex files match the way LFS files are rendered, making annexed files accessible via the web instead of being black boxes only accessible by git clone. This mostly just duplicates the existing LFS logic. It doesn't try to combine itself with the existing logic, to make merging with upstream easier. If upstream ever decides to accept, I would like to try to merge the redundant logic. The one bit that doesn't directly copy LFS is my choice to hide annex-symlinks. LFS files are always _pointer files_ and therefore always render with the "file" icon and no special label, but annex files come in two flavours: symlinks or pointer files. I've conflated both kinds to try to give a consistent experience. The tests in here ensure the correct download link (/media, from the last PR) renders in both the toolbar and, if a binary file (like most annexed files will be), in the main pane, but it also adds quite a bit of code to make sure text files that happen to be annexed are dug out and rendered inline like LFS files are. --- modules/base/tool.go | 7 ++ options/locale/locale_cs-CZ.ini | 2 + options/locale/locale_de-DE.ini | 2 + options/locale/locale_el-GR.ini | 2 + options/locale/locale_en-US.ini | 2 + options/locale/locale_es-ES.ini | 2 + options/locale/locale_fa-IR.ini | 2 + options/locale/locale_fr-FR.ini | 2 + options/locale/locale_hu-HU.ini | 2 + options/locale/locale_id-ID.ini | 2 + options/locale/locale_is-IS.ini | 1 + options/locale/locale_it-IT.ini | 2 + options/locale/locale_ja-JP.ini | 2 + options/locale/locale_ko-KR.ini | 1 + options/locale/locale_lv-LV.ini | 4 +- options/locale/locale_nl-NL.ini | 2 + options/locale/locale_pl-PL.ini | 2 + options/locale/locale_pt-BR.ini | 2 + options/locale/locale_pt-PT.ini | 2 + options/locale/locale_ru-RU.ini | 2 + options/locale/locale_si-LK.ini | 2 + options/locale/locale_sk-SK.ini | 1 + options/locale/locale_sv-SE.ini | 2 + options/locale/locale_tr-TR.ini | 2 + options/locale/locale_uk-UA.ini | 2 + options/locale/locale_zh-CN.ini | 2 + options/locale/locale_zh-HK.ini | 3 +- options/locale/locale_zh-TW.ini | 2 + routers/web/repo/view.go | 66 +++++++++++--- templates/repo/file_info.tmpl | 1 + tests/integration/git_annex_test.go | 133 ++++++++++++++++++++++++++++ 31 files changed, 248 insertions(+), 13 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index fd6a7c2b77..e70d7d4df7 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -16,6 +16,7 @@ import ( "strings" "unicode/utf8" + "forgejo.org/modules/annex" "forgejo.org/modules/git" "forgejo.org/modules/log" @@ -103,6 +104,12 @@ func Int64sToStrings(ints []int64) []string { func EntryIcon(entry *git.TreeEntry) string { switch { case entry.IsLink(): + isAnnexed, _ := annex.IsAnnexed(entry.Blob()) + if isAnnexed { + // git-annex files are sometimes stored as symlinks; + // short-circuit that so like LFS they are displayed as regular files + return "file" + } te, _, err := entry.FollowLink() if err != nil { log.Debug(err.Error()) diff --git a/options/locale/locale_cs-CZ.ini b/options/locale/locale_cs-CZ.ini index e61c3fa90d..d342d5fcb2 100644 --- a/options/locale/locale_cs-CZ.ini +++ b/options/locale/locale_cs-CZ.ini @@ -1349,6 +1349,7 @@ view_git_blame=Zobrazit git blame video_not_supported_in_browser=Váš prohlížeč nepodporuje značku HTML5 „video“. audio_not_supported_in_browser=Váš prohlížeč nepodporuje značku HTML5 „audio“. stored_lfs=Uloženo pomocí Git LFS +stored_annex=Uloženo pomocí Git Annex symbolic_link=Symbolický odkaz executable_file=Spustitelný soubor vendored = Vendorováno @@ -1374,6 +1375,7 @@ editor.upload_file=Nahrát soubor editor.edit_file=Upravit soubor editor.preview_changes=Náhled změn editor.cannot_edit_lfs_files=Soubory LFS nemohou být upravovány přes webové rozhraní. +editor.cannot_edit_annex_files=Annex soubory nemohou být upravovány přes webové rozhraní. editor.cannot_edit_non_text_files=Binární soubory nemohou být upravovány přes webové rozhraní. editor.edit_this_file=Upravit soubor editor.this_file_locked=Soubor je uzamčen diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index c4801a2fef..26f24ab6c7 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -1349,6 +1349,7 @@ view_git_blame=„git blame“ ansehen video_not_supported_in_browser=Dein Browser unterstützt das HTML5-„video“-Tag nicht. audio_not_supported_in_browser=Dein Browser unterstützt das HTML5-„audio“-Tag nicht. stored_lfs=Gespeichert mit Git LFS +stored_annex=Gespeichert mit Git Annex symbolic_link=Softlink executable_file=Ausführbare Datei commit_graph=Commit-Graph @@ -1372,6 +1373,7 @@ editor.upload_file=Datei hochladen editor.edit_file=Datei bearbeiten editor.preview_changes=Vorschau der Änderungen editor.cannot_edit_lfs_files=LFS-Dateien können im Webinterface nicht bearbeitet werden. +editor.cannot_edit_annex_files=Annex-Dateien können im Webinterface nicht bearbeitet werden. editor.cannot_edit_non_text_files=Binärdateien können nicht im Webinterface bearbeitet werden. editor.edit_this_file=Datei bearbeiten editor.this_file_locked=Datei ist gesperrt diff --git a/options/locale/locale_el-GR.ini b/options/locale/locale_el-GR.ini index c20a2f3172..4eb1322cea 100644 --- a/options/locale/locale_el-GR.ini +++ b/options/locale/locale_el-GR.ini @@ -1321,6 +1321,7 @@ view_git_blame=Προβολή git blame video_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 «video». audio_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 «audio». stored_lfs=Αποθηκεύτηκε με το Git LFS +stored_annex=Αποθηκεύτηκε με το Git Annex symbolic_link=Symbolic link executable_file=Εκτελέσιμο αρχείο commit_graph=Γράφημα commit @@ -1344,6 +1345,7 @@ editor.upload_file=Ανέβασμα αρχείου editor.edit_file=Επεξεργασία αρχείου editor.preview_changes=Προεπισκόπηση αλλαγών editor.cannot_edit_lfs_files=Τα αρχεία LFS δεν μπορούν να επεξεργαστούν στη διεπαφή web. +editor.cannot_edit_annex_files=Τα αρχεία Annex δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.cannot_edit_non_text_files=Τα δυαδικά αρχεία δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.edit_this_file=Επεξεργασία αρχείου editor.this_file_locked=Το αρχείο είναι κλειδωμένο diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index bfdcdb9112..5acf932e23 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1367,6 +1367,7 @@ view_git_blame = View git blame video_not_supported_in_browser = Your browser does not support the HTML5 "video" tag. audio_not_supported_in_browser = Your browser does not support the HTML5 "audio" tag. stored_lfs = Stored with Git LFS +stored_annex = Stored with Git Annex symbolic_link = Symbolic link executable_file = Executable file vendored = Vendored @@ -1394,6 +1395,7 @@ editor.upload_file = Upload file editor.edit_file = Edit file editor.preview_changes = Preview changes editor.cannot_edit_lfs_files = LFS files cannot be edited in the web interface. +editor.cannot_edit_annex_files = Annex files cannot be edited in the web interface. editor.cannot_edit_non_text_files = Binary files cannot be edited in the web interface. editor.edit_this_file = Edit file editor.this_file_locked = File is locked diff --git a/options/locale/locale_es-ES.ini b/options/locale/locale_es-ES.ini index 3ff6c565b7..180dade48f 100644 --- a/options/locale/locale_es-ES.ini +++ b/options/locale/locale_es-ES.ini @@ -1340,6 +1340,7 @@ view_git_blame=Ver Git blame video_not_supported_in_browser=Su navegador no soporta el tag "video" de HTML5. audio_not_supported_in_browser=Su navegador no soporta el tag "audio" de HTML5. stored_lfs=Almacenados con Git LFS +stored_annex=Almacenados con Git Annex symbolic_link=Enlace simbólico executable_file=Archivo ejecutable commit_graph=Gráfico de commits @@ -1363,6 +1364,7 @@ editor.upload_file=Subir archivo editor.edit_file=Editar archivo editor.preview_changes=Vista previa de los cambios editor.cannot_edit_lfs_files=Los archivos LFS no se pueden editar en la interfaz web. +editor.cannot_edit_annex_files=Los archivos Annex no se pueden editar en la interfaz web. editor.cannot_edit_non_text_files=Los archivos binarios no se pueden editar en la interfaz web. editor.edit_this_file=Editar archivo editor.this_file_locked=El archivo está bloqueado diff --git a/options/locale/locale_fa-IR.ini b/options/locale/locale_fa-IR.ini index 804b48b2b2..6b6a530639 100644 --- a/options/locale/locale_fa-IR.ini +++ b/options/locale/locale_fa-IR.ini @@ -1036,6 +1036,7 @@ file_copy_permalink=پرمالینک را کپی کنید video_not_supported_in_browser=مرورگر شما از تگ video که در HTML5 تعریف شده است، پشتیبانی نمی کند. audio_not_supported_in_browser=مرورگر شما از تگ audio که در HTML5 تعریف شده است، پشتیبانی نمی کند. stored_lfs=ذخیره شده با GIT LFS +stored_annex=ذخیره شده با GIT Annex symbolic_link=پیوند نمادین commit_graph=نمودار کامیت commit_graph.select=انتخاب برنچها @@ -1053,6 +1054,7 @@ editor.upload_file=بارگذاری پرونده editor.edit_file=ویرایش پرونده editor.preview_changes=پیش نمایش تغییرات editor.cannot_edit_lfs_files=پرونده های LFS در صحفه وب قابل تغییر نیست. +editor.cannot_edit_annex_files=پرونده های Annex در صحفه وب قابل تغییر نیست. editor.cannot_edit_non_text_files=پرونده‎های دودویی در صفحه وب قابل تغییر نیست. editor.edit_this_file=ویرایش پرونده editor.this_file_locked=پرونده قفل شده است diff --git a/options/locale/locale_fr-FR.ini b/options/locale/locale_fr-FR.ini index ef4b7bad5d..717a204f8e 100644 --- a/options/locale/locale_fr-FR.ini +++ b/options/locale/locale_fr-FR.ini @@ -1351,6 +1351,7 @@ view_git_blame=Voir Git blame video_not_supported_in_browser=Votre navigateur ne supporte pas la balise « vidéo » HTML5. audio_not_supported_in_browser=Votre navigateur ne supporte pas la balise « audio » HTML5. stored_lfs=Stocké avec Git LFS +stored_annex=Stocké avec Git Annex symbolic_link=Lien symbolique executable_file=Fichier exécutable vendored = Vendored @@ -1376,6 +1377,7 @@ editor.upload_file=Téléverser un fichier editor.edit_file=Modifier le fichier editor.preview_changes=Aperçu des modifications editor.cannot_edit_lfs_files=Les fichiers LFS ne peuvent pas être modifiés dans l'interface web. +editor.cannot_edit_annex_files=Les fichiers Annex ne peuvent pas être modifiés dans l'interface web. editor.cannot_edit_non_text_files=Les fichiers binaires ne peuvent pas être édités dans l'interface web. editor.edit_this_file=Modifier le fichier editor.this_file_locked=Le fichier est verrouillé diff --git a/options/locale/locale_hu-HU.ini b/options/locale/locale_hu-HU.ini index 411bad835a..ea8732f422 100644 --- a/options/locale/locale_hu-HU.ini +++ b/options/locale/locale_hu-HU.ini @@ -787,6 +787,7 @@ file_too_large=Ez a fájl túl nagy ahhoz, hogy megjelenítsük. video_not_supported_in_browser=A böngésző nem támogatja a HTML5 video tag-et. audio_not_supported_in_browser=A böngésző nem támogatja a HTML5 audio tag-et. stored_lfs=Git LFS-el eltárolva +stored_annex=Git Annex-el eltárolva symbolic_link=Szimbolikus hivatkozás commit_graph=Commit gráf commit_graph.hide_pr_refs=Pull request-ek elrejtése @@ -799,6 +800,7 @@ editor.upload_file=Fájl feltöltése editor.edit_file=Fájl szerkesztése editor.preview_changes=Változások előnézete editor.cannot_edit_lfs_files=LFS fájlok nem szerkeszthetőek a webes felületen. +editor.cannot_edit_annex_files=Annex fájlok nem szerkeszthetőek a webes felületen. editor.cannot_edit_non_text_files=Bináris fájlok nem szerkeszthetőek a webes felületen. editor.edit_this_file=Fájl szerkesztése editor.this_file_locked=Zárolt állomány diff --git a/options/locale/locale_id-ID.ini b/options/locale/locale_id-ID.ini index 673d1464b1..fc9d934ce6 100644 --- a/options/locale/locale_id-ID.ini +++ b/options/locale/locale_id-ID.ini @@ -681,6 +681,7 @@ file_permalink=Permalink file_too_large=Berkas terlalu besar untuk ditampilkan. stored_lfs=Tersimpan dengan GIT LFS +stored_annex=Tersimpan dengan GIT Annex commit_graph=Grafik Komit blame=Salahkan normal_view=Pandangan Normal @@ -692,6 +693,7 @@ editor.upload_file=Unggah Berkas editor.edit_file=Sunting Berkas editor.preview_changes=Tinjau Perubahan editor.cannot_edit_lfs_files=Berkas LFS tidak dapat disunting dalam antarmuka web. +editor.cannot_edit_annex_files=Berkas Annex tidak dapat disunting dalam antarmuka web. editor.cannot_edit_non_text_files=Berkas biner tidak dapat disunting dalam antarmuka web. editor.edit_this_file=Sunting Berkas editor.this_file_locked=Berkas terkunci diff --git a/options/locale/locale_is-IS.ini b/options/locale/locale_is-IS.ini index 9b1d56fed9..9c39b9d830 100644 --- a/options/locale/locale_is-IS.ini +++ b/options/locale/locale_is-IS.ini @@ -684,6 +684,7 @@ file_view_rendered=Skoða Unnið file_copy_permalink=Afrita Varanlega Slóð stored_lfs=Geymt með Git LFS +stored_annex=Geymt með Git Annex commit_graph.hide_pr_refs=Fela Sameiningarbeiðnir commit_graph.monochrome=Einlitað commit_graph.color=Litað diff --git a/options/locale/locale_it-IT.ini b/options/locale/locale_it-IT.ini index c4083f0ce8..3c5e834260 100644 --- a/options/locale/locale_it-IT.ini +++ b/options/locale/locale_it-IT.ini @@ -1277,6 +1277,7 @@ view_git_blame=Visualizza git incolpa video_not_supported_in_browser=Il tuo browser non supporta le etichette "video" di HTML5. audio_not_supported_in_browser=Il tuo browser non supporta le etichette "audio" di HTML5. stored_lfs=Memorizzati con Git LFS +stored_annex=Memorizzati con Git Annex symbolic_link=Link Simbolico commit_graph=Grafico dei commit commit_graph.select=Seleziona rami @@ -1295,6 +1296,7 @@ editor.upload_file=Carica file editor.edit_file=Modifica file editor.preview_changes=Anteprima modifiche editor.cannot_edit_lfs_files=I file LFS non possono essere modificati nell'interfaccia web. +editor.cannot_edit_annex_files=I file Annex non possono essere modificati nell'interfaccia web. editor.cannot_edit_non_text_files=I file binari non possono essere modificati tramite interfaccia web. editor.edit_this_file=Modifica file editor.this_file_locked=Il file è bloccato diff --git a/options/locale/locale_ja-JP.ini b/options/locale/locale_ja-JP.ini index d4d7024f5d..32caac1371 100644 --- a/options/locale/locale_ja-JP.ini +++ b/options/locale/locale_ja-JP.ini @@ -1307,6 +1307,7 @@ view_git_blame=Git Blameを表示 video_not_supported_in_browser=このブラウザはHTML5のvideoタグをサポートしていません。 audio_not_supported_in_browser=このブラウザーはHTML5のaudioタグをサポートしていません。 stored_lfs=Git LFSで保管されています +stored_annex=Git Annexで保管されています symbolic_link=シンボリック リンク executable_file=実行ファイル commit_graph=コミットグラフ @@ -1330,6 +1331,7 @@ editor.upload_file=ファイルをアップロード editor.edit_file=ファイルを編集 editor.preview_changes=変更をプレビュー editor.cannot_edit_lfs_files=LFSのファイルはWebインターフェースで編集できません。 +editor.cannot_edit_annex_files=AnnexのファイルはWebインターフェースで編集できません。 editor.cannot_edit_non_text_files=バイナリファイルはWebインターフェースで編集できません。 editor.edit_this_file=ファイルを編集 editor.this_file_locked=ファイルはロックされています diff --git a/options/locale/locale_ko-KR.ini b/options/locale/locale_ko-KR.ini index 5e31ae4f19..7e7871777d 100644 --- a/options/locale/locale_ko-KR.ini +++ b/options/locale/locale_ko-KR.ini @@ -811,6 +811,7 @@ file_too_large=보여주기에는 파일이 너무 큽니다. video_not_supported_in_browser=당신의 브라우저가 HTML5의 "video" 태그를 지원하지 않습니다. audio_not_supported_in_browser=당신의 브라우저가 HTML5의 "audio" 태그를 지원하지 않습니다. stored_lfs=Git LFS에 저장되어 있습니다 +stored_annex=Git Annex에 저장되어 있습니다 commit_graph=커밋 그래프 editor.new_file=새 파일 diff --git a/options/locale/locale_lv-LV.ini b/options/locale/locale_lv-LV.ini index 70ce31ec4f..e4b51bbfb7 100644 --- a/options/locale/locale_lv-LV.ini +++ b/options/locale/locale_lv-LV.ini @@ -1348,6 +1348,7 @@ view_git_blame=Apskatīt Git izmaiņu veicējus video_not_supported_in_browser=Pārlūks neatbalsta HTML5 tagu "video". audio_not_supported_in_browser=Pārlūks neatbalsta HTML5 tagu "audio". stored_lfs=Saglabāts Git LFS +stored_annex=Saglabāts Git Annex symbolic_link=Simboliska saite executable_file=Izpildāma datne commit_graph=Iesūtījumu karte @@ -1371,6 +1372,7 @@ editor.upload_file=Augšupielādēt datni editor.edit_file=Labot datni editor.preview_changes=Priekšskatīt izmaiņas editor.cannot_edit_lfs_files=LFS datnes nevar labot tīmekļa saskarnē. +editor.cannot_edit_annex_files=Annex datnes tīmekļa saskarnē nevar labot. editor.cannot_edit_non_text_files=Binārās datnes nevar labot tīmekļa saskarnē. editor.edit_this_file=Labot datni editor.this_file_locked=Datne ir slēgta @@ -4072,4 +4074,4 @@ filepreview.lines = %[1]d. līdz %[2]d. rinda %[3]s filepreview.truncated = Priekšskatījums tika saīsināts [translation_meta] -test = Šī ir pārbaudes virkne. Tā netiek attēlota Forgejo saskarnē, bet tiek izmantota pārbaudes nolūkiem. Droši var ievadīt "ok", lai ietaupītu laiku (vai kādu jautru faktu pēc izvēles), lai sasniegtu to saldo 100% pabeigšanas atzīmi. \ No newline at end of file +test = Šī ir pārbaudes virkne. Tā netiek attēlota Forgejo saskarnē, bet tiek izmantota pārbaudes nolūkiem. Droši var ievadīt "ok", lai ietaupītu laiku (vai kādu jautru faktu pēc izvēles), lai sasniegtu to saldo 100% pabeigšanas atzīmi. diff --git a/options/locale/locale_nl-NL.ini b/options/locale/locale_nl-NL.ini index bfd6c6878e..f68f738729 100644 --- a/options/locale/locale_nl-NL.ini +++ b/options/locale/locale_nl-NL.ini @@ -1316,6 +1316,7 @@ view_git_blame=Bekijk git blame video_not_supported_in_browser=Uw browser ondersteunt de HTML5 "video" element niet. audio_not_supported_in_browser=Uw browser ondersteunt de HTML5 "audio" element niet. stored_lfs=Opgeslagen met Git LFS +stored_annex=Opgeslagen met Git Annex symbolic_link=Symbolische link commit_graph=Commit grafiek commit_graph.select=Selecteer branches @@ -1334,6 +1335,7 @@ editor.upload_file=Upload bestand editor.edit_file=Bewerk bestand editor.preview_changes=Voorbeeld tonen editor.cannot_edit_lfs_files=LFS-bestanden kunnen niet worden bewerkt in de webinterface. +editor.cannot_edit_annex_files=Annex-bestanden kunnen niet worden bewerkt in de webinterface. editor.cannot_edit_non_text_files=Binaire bestanden kunnen niet worden bewerkt in de webinterface. editor.edit_this_file=Bewerk bestand editor.this_file_locked=Bestand is vergrendeld diff --git a/options/locale/locale_pl-PL.ini b/options/locale/locale_pl-PL.ini index 8aaa680009..640e031956 100644 --- a/options/locale/locale_pl-PL.ini +++ b/options/locale/locale_pl-PL.ini @@ -1255,6 +1255,7 @@ file_copy_permalink=Kopiuj bezpośredni odnośnik video_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "video". audio_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "audio". stored_lfs=Przechowane za pomocą Git LFS +stored_annex=Przechowane za pomocą Git Annex symbolic_link=Dowiązanie symboliczne commit_graph=Wykres commitów commit_graph.select=Wybierz gałęzie @@ -1272,6 +1273,7 @@ editor.upload_file=Wyślij plik editor.edit_file=Edytuj plik editor.preview_changes=Podgląd zmian editor.cannot_edit_lfs_files=Pliki LFS nie mogą być edytowane poprzez interfejs przeglądarkowy. +editor.cannot_edit_annex_files=Pliki Annex nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.cannot_edit_non_text_files=Pliki binarne nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.edit_this_file=Edytuj plik editor.this_file_locked=Plik jest zablokowany diff --git a/options/locale/locale_pt-BR.ini b/options/locale/locale_pt-BR.ini index 92fa50d618..2bfe2613bf 100644 --- a/options/locale/locale_pt-BR.ini +++ b/options/locale/locale_pt-BR.ini @@ -1341,6 +1341,7 @@ view_git_blame=Ver git blame video_not_supported_in_browser=Seu navegador não tem suporte para a tag "video" do HTML5. audio_not_supported_in_browser=Seu navegador não tem suporte para a tag "audio" do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Link simbólico executable_file=Arquivo executável commit_graph=Gráfico de commits @@ -1364,6 +1365,7 @@ editor.upload_file=Enviar arquivo editor.edit_file=Editar arquivo editor.preview_changes=Pré-visualizar alterações editor.cannot_edit_lfs_files=Arquivos LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Arquivos Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Arquivos binários não podem ser editados na interface web. editor.edit_this_file=Editar arquivo editor.this_file_locked=Arquivo está bloqueado diff --git a/options/locale/locale_pt-PT.ini b/options/locale/locale_pt-PT.ini index 05613030b5..eea29f8850 100644 --- a/options/locale/locale_pt-PT.ini +++ b/options/locale/locale_pt-PT.ini @@ -1353,6 +1353,7 @@ view_git_blame=Ver git blame video_not_supported_in_browser=O seu navegador não suporta a etiqueta "video" do HTML5. audio_not_supported_in_browser=O seu navegador não suporta a etiqueta "audio" do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Ligação simbólica executable_file=Ficheiro executável vendored=Externo @@ -1378,6 +1379,7 @@ editor.upload_file=Carregar ficheiro editor.edit_file=Editar ficheiro editor.preview_changes=Pré-visualizar modificações editor.cannot_edit_lfs_files=Ficheiros LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Ficheiros Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Ficheiros binários não podem ser editados na interface da web. editor.edit_this_file=Editar ficheiro editor.this_file_locked=Ficheiro bloqueado diff --git a/options/locale/locale_ru-RU.ini b/options/locale/locale_ru-RU.ini index 38d1d885cc..1d5c115e67 100644 --- a/options/locale/locale_ru-RU.ini +++ b/options/locale/locale_ru-RU.ini @@ -1336,6 +1336,7 @@ view_git_blame=Показать git blame video_not_supported_in_browser=Ваш браузер не поддерживает тэг HTML5 «video». audio_not_supported_in_browser=Ваш браузер не поддерживает тэг HTML5 «audio». stored_lfs=Хранится Git LFS +stored_annex=Хранится Git Annex symbolic_link=Символическая ссылка executable_file=Исполняемый файл commit_graph=Граф коммитов @@ -1359,6 +1360,7 @@ editor.upload_file=Загрузить файл editor.edit_file=Редактировать файл editor.preview_changes=Просмотр изменений editor.cannot_edit_lfs_files=LFS файлы невозможно редактировать в веб-интерфейсе. +editor.cannot_edit_annex_files=Annex файлы невозможно редактировать в веб-интерфейсе. editor.cannot_edit_non_text_files=Двоичные файлы нельзя редактировать в веб-интерфейсе. editor.edit_this_file=Редактировать файл editor.this_file_locked=Файл заблокирован diff --git a/options/locale/locale_si-LK.ini b/options/locale/locale_si-LK.ini index d55b238b1c..6dbb6dc3c2 100644 --- a/options/locale/locale_si-LK.ini +++ b/options/locale/locale_si-LK.ini @@ -894,6 +894,7 @@ file_copy_permalink=පිටපත් මාමලින්ක් video_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'වීඩියෝ' ටැගය සඳහා සහය නොදක්වයි. audio_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'ශ්රව්ය' ටැගය සඳහා සහය නොදක්වයි. stored_lfs=Git LFS සමඟ ගබඩා +stored_annex=Git Annex සමඟ ගබඩා symbolic_link=සංකේතාත්මක සබැඳිය commit_graph=ප්රස්තාරය කැප commit_graph.select=ශාඛා තෝරන්න @@ -911,6 +912,7 @@ editor.upload_file=ගොනුව උඩුගත කරන්න editor.edit_file=ගොනුව සංස්කරණය editor.preview_changes=වෙනස්කම් පෙරදසුන editor.cannot_edit_lfs_files=LFS ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. +editor.cannot_edit_annex_files=Annex ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.cannot_edit_non_text_files=ද්විමය ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.edit_this_file=ගොනුව සංස්කරණය editor.this_file_locked=ගොනුවට අගුළු ලා ඇත diff --git a/options/locale/locale_sk-SK.ini b/options/locale/locale_sk-SK.ini index a90ddd513b..e43ba1ce37 100644 --- a/options/locale/locale_sk-SK.ini +++ b/options/locale/locale_sk-SK.ini @@ -1013,6 +1013,7 @@ view_git_blame=Zobraziť Git Blame video_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'video'. audio_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'audio'. stored_lfs=Uložené pomocou Git LFS +stored_annex=Uložené pomocou Git Annex symbolic_link=Symbolický odkaz commit_graph=Graf commitov line=riadok diff --git a/options/locale/locale_sv-SE.ini b/options/locale/locale_sv-SE.ini index cbb42b3caa..96e2ef2b84 100644 --- a/options/locale/locale_sv-SE.ini +++ b/options/locale/locale_sv-SE.ini @@ -907,6 +907,7 @@ file_too_large=Filen är för stor för att visas. video_not_supported_in_browser=Din webbläsare stödjer ej HTML5-taggen "video". audio_not_supported_in_browser=Din webbläsare stödjer ej HTML5-taggen "audio". stored_lfs=Sparad med Git LFS +stored_annex=Sparad med Git Annex symbolic_link=Symbolisk länk commit_graph=Commitgraf commit_graph.monochrome=Mono @@ -920,6 +921,7 @@ editor.upload_file=Ladda upp fil editor.edit_file=Redigera fil editor.preview_changes=Förhandsgranska ändringar editor.cannot_edit_lfs_files=LFS-filer kan inte redigeras i webbgränssnittet. +editor.cannot_edit_annex_files=Annex-filer kan inte redigeras i webbgränssnittet. editor.cannot_edit_non_text_files=Binära filer kan inte redigeras genom webbgränssnittet. editor.edit_this_file=Redigera fil editor.this_file_locked=Filen är låst diff --git a/options/locale/locale_tr-TR.ini b/options/locale/locale_tr-TR.ini index 2743f63a41..b9fbc9c52f 100644 --- a/options/locale/locale_tr-TR.ini +++ b/options/locale/locale_tr-TR.ini @@ -1289,6 +1289,7 @@ view_git_blame=Git Suç Görüntüle video_not_supported_in_browser=Tarayıcınız HTML5 'video' etiketini desteklemiyor. audio_not_supported_in_browser=Tarayıcınız HTML5 'audio' etiketini desteklemiyor. stored_lfs=Git LFS ile depolandı +stored_annex=Git Annex ile depolandı symbolic_link=Sembolik Bağlantı executable_file=Çalıştırılabilir Dosya commit_graph=İşleme Grafiği @@ -1312,6 +1313,7 @@ editor.upload_file=Dosya Yükle editor.edit_file=Dosyayı Düzenle editor.preview_changes=Değişiklikleri Önizle editor.cannot_edit_lfs_files=LFS dosyaları web arayüzünde düzenlenemez. +editor.cannot_edit_annex_files=Annex dosyaları web arayüzünde düzenlenemez. editor.cannot_edit_non_text_files=Bu tür dosyalar web arayüzünden düzenlenemez. editor.edit_this_file=Dosyayı Düzenle editor.this_file_locked=Dosya kilitlendi diff --git a/options/locale/locale_uk-UA.ini b/options/locale/locale_uk-UA.ini index 7d3d6f4c8b..0bd3a937a4 100644 --- a/options/locale/locale_uk-UA.ini +++ b/options/locale/locale_uk-UA.ini @@ -1285,6 +1285,7 @@ file_copy_permalink=Копіювати постійне посилання video_not_supported_in_browser=Ваш браузер не підтримує тег HTML5 «video». audio_not_supported_in_browser=Ваш браузер не підтримує тег HTML5 «audio». stored_lfs=Збережено з Git LFS +stored_annex=Збережено з Git Annex symbolic_link=Символічне посилання commit_graph=Графік комітів commit_graph.select=Виберіть гілки @@ -1302,6 +1303,7 @@ editor.upload_file=Завантажити файл editor.edit_file=Редагувати файл editor.preview_changes=Попередній перегляд змін editor.cannot_edit_lfs_files=Файли LFS не можна редагувати в веб-інтерфейсі. +editor.cannot_edit_annex_files=Файли Annex не можна редагувати в веб-інтерфейсі. editor.cannot_edit_non_text_files=Бінарні файли не можливо редагувати у веб-інтерфейсі. editor.edit_this_file=Редагувати файл editor.this_file_locked=Файл заблоковано diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2e062d80b4..226f1be3b0 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1351,6 +1351,7 @@ view_git_blame=查看 Git Blame video_not_supported_in_browser=您的浏览器不支持 HTML5 “video” 标签。 audio_not_supported_in_browser=您的浏览器不支持 HTML5 “audio” 标签。 stored_lfs=存储到Git LFS +stored_annex=存储到Git Annex symbolic_link=符号链接 executable_file=可执行文件 vendored = Vendored @@ -1376,6 +1377,7 @@ editor.upload_file=上传文件 editor.edit_file=编辑文件 editor.preview_changes=预览变更 editor.cannot_edit_lfs_files=无法在 web 界面中编辑 lfs 文件。 +editor.cannot_edit_annex_files=无法在 web 界面中编辑 lfs 文件。 editor.cannot_edit_non_text_files=网页不能编辑二进制文件。 editor.edit_this_file=编辑文件 editor.this_file_locked=文件已锁定 diff --git a/options/locale/locale_zh-HK.ini b/options/locale/locale_zh-HK.ini index e2cb0d8b2c..93744cbbcf 100644 --- a/options/locale/locale_zh-HK.ini +++ b/options/locale/locale_zh-HK.ini @@ -488,6 +488,7 @@ file_view_raw=查看原始文件 file_permalink=永久連結 stored_lfs=儲存到到 Git LFS +stored_annex=儲存到到 Git Annex editor.preview_changes=預覽更改 editor.or=或 @@ -1154,4 +1155,4 @@ runners.labels = 標籤 [git.filemode] -[search] \ No newline at end of file +[search] diff --git a/options/locale/locale_zh-TW.ini b/options/locale/locale_zh-TW.ini index ea8c1bc2b1..51c9ba7139 100644 --- a/options/locale/locale_zh-TW.ini +++ b/options/locale/locale_zh-TW.ini @@ -1292,6 +1292,7 @@ view_git_blame=檢視 Git Blame video_not_supported_in_browser=您的瀏覽器不支援 HTML5 的「video」標籤。 audio_not_supported_in_browser=您的瀏覽器不支援 HTML5 的「audio」標籤。 stored_lfs=已使用 Git LFS 儲存 +stored_annex=已使用 Git Annex 儲存 symbolic_link=符號連結 commit_graph=提交線圖 commit_graph.select=選擇分支 @@ -1311,6 +1312,7 @@ editor.upload_file=上傳檔案 editor.edit_file=編輯檔案 editor.preview_changes=預覽變更 editor.cannot_edit_lfs_files=無法在 web 介面中編輯 LFS 檔。 +editor.cannot_edit_annex_files=無法在 web 介面中編輯 Annex 檔。 editor.cannot_edit_non_text_files=網站介面不能編輯二進位檔案。 editor.edit_this_file=編輯檔案 editor.this_file_locked=檔案已被鎖定 diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index bea002f690..380218ec31 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -34,6 +34,7 @@ import ( unit_model "forgejo.org/models/unit" user_model "forgejo.org/models/user" "forgejo.org/modules/actions" + "forgejo.org/modules/annex" "forgejo.org/modules/base" "forgejo.org/modules/charset" "forgejo.org/modules/git" @@ -209,14 +210,47 @@ func localizedExtensions(ext, languageCode string) (localizedExts []string) { } type fileInfo struct { - isTextFile bool - isLFSFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + isTextFile bool + isLFSFile bool + isAnnexFile bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + return nil, nil, nil, err + } + if isAnnexed { + // TODO: this code could be merged with the LFS case, especially the redundant type sniffer, + // but it is *currently* written this way to make merging with the non-annex upstream easier: + // this way, the git-annex patch is (mostly) pure additions. + + annexContent, err := annex.Content(blob) + if err != nil { + // in the case where annex content is missing, what should happen? + // do we render the page with an error message? + // actually that's not a bad idea, there's some sort of error message situation + // TODO: display an error to the user explaining that their data is missing + return nil, nil, nil, err + } + + stat, err := annexContent.Stat() + if err != nil { + return nil, nil, nil, err + } + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(annexContent, buf) + buf = buf[:n] + + st := typesniffer.DetectContentType(buf) + + return buf, annexContent, &fileInfo{st.IsText(), false, true, stat.Size(), nil, st}, nil + } + dataRc, err := blob.DataAsync() if err != nil { return nil, nil, nil, err @@ -231,18 +265,18 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, // FIXME: what happens when README file is an image? if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) if err != nil { // fallback to plain file log.Warn("Unable to access LFS pointer %s in repo %d: %v", pointer.Oid, repoID, err) - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } dataRc.Close() @@ -262,7 +296,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st = typesniffer.DetectContentType(buf) - return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil + return buf, dataRc, &fileInfo{st.IsText(), true, false, meta.Size, &meta.Pointer, st}, nil } func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.TreeEntry) { @@ -451,10 +485,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { isDisplayingSource := ctx.FormString("display") == "source" isDisplayingRendered := !isDisplayingSource - if fInfo.isLFSFile { + if fInfo.isLFSFile || fInfo.isAnnexFile { ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } + if fInfo.isAnnexFile { + // pre-git-annex v7, all annexed files were represented in-repo as symlinks; + // but we pretend they aren't, since that's a distracting quirk of git-annex + // and not a meaningful choice on the user's part + ctx.Data["FileIsSymlink"] = false + } + isRepresentableAsText := fInfo.st.IsRepresentableAsText() if !isRepresentableAsText { // If we can't show plain text, always try to render. @@ -462,6 +503,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { isDisplayingRendered = true } ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["IsAnnexFile"] = fInfo.isAnnexFile ctx.Data["FileSize"] = fInfo.fileSize ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText @@ -496,6 +538,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { // Assume file is not editable first. if fInfo.isLFSFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") + } else if fInfo.isAnnexFile { + ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_annex_files") } else if !isRepresentableAsText { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") } @@ -603,7 +647,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { ctx.Data["FileContent"] = fileContent ctx.Data["LineEscapeStatus"] = statuses } - if !fInfo.isLFSFile { + if !fInfo.isLFSFile && !fInfo.isAnnexFile { if ctx.Repo.CanEnableEditor(ctx, ctx.Doer) { if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID { ctx.Data["CanEditFile"] = false diff --git a/templates/repo/file_info.tmpl b/templates/repo/file_info.tmpl index 6ae7c15a26..05d9825cfb 100644 --- a/templates/repo/file_info.tmpl +++ b/templates/repo/file_info.tmpl @@ -17,6 +17,7 @@ {{if .FileSize}}
{{ctx.Locale.TrSize .FileSize}}{{if .IsLFSFile}} ({{ctx.Locale.Tr "repo.stored_lfs"}}){{end}} + {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}){{end}}
{{end}} {{if .LFSLock}} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 3e54ace2f2..2a016c61dc 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -118,6 +118,111 @@ func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { require.True(t, match, "Annexed files should be the same") } +func TestGitAnnexViews(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-template-render-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + session := loginUser(t, ctx.Username) + + t.Run("Index", func(t *testing.T) { + // test that annex symlinks renders with the _file icon_ on the main list + defer tests.PrintCurrentTest(t)() + + repoLink := path.Join("/", ctx.Username, ctx.Reponame) + req := NewRequest(t, "GET", repoLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc := NewHTMLParser(t, resp.Body) + isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") + require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") + }) + + t.Run("View", func(t *testing.T) { + // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files + defer tests.PrintCurrentTest(t)() + + doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { + viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) + // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? + mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) + + req := NewRequest(t, "GET", viewLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc = NewHTMLParser(t, resp.Body) + // the first button on the toolbar on the view template is the "Raw" button + // this CSS selector is the most precise I can think to use + buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") + require.True(t, exists, "Download button should exist on the file header") + require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") + + return htmlDoc, viewLink, mediaLink + } + + t.Run("Binary", func(t *testing.T) { + // test that annexing a file renders the /media link in /src and NOT the /raw link + defer tests.PrintCurrentTest(t)() + + doBinaryViewTest := func(file string) { + htmlDoc, _, mediaLink := doViewTest(file) + + rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") + require.True(t, exists, "Download link should render instead of content because this is a binary file") + require.EqualValues(t, mediaLink, rawLink) + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.bin") + }) + }) + + t.Run("Text", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + doTextViewTest := func(file string) { + htmlDoc, _, _ := doViewTest(file) + require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.txt") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.md") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.rst") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.markdown") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + }) + }) + }) +} + /* Test that permissions are enforced on git-annex-shell commands. @@ -1029,6 +1134,14 @@ func doInitAnnexRepository(repoPath string) error { if err != nil { return err } + _, err = f.WriteString("*.rst filter=annex\n") + if err != nil { + return err + } + _, err = f.WriteString("*.markdown filter=annex\n") + if err != nil { + return err + } f.Close() err = git.AddChanges(repoPath, false, ".") @@ -1053,6 +1166,18 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.md"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.txt"), []byte("We're going to see the wizard\nThe wonderful\nMonkey of\nBoz\n"), 0o777) + if err != nil { + return err + } + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err @@ -1065,6 +1190,14 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.rst"), []byte("Title\n=====\n\n- this is to test annexing a text file\n- lists are fun\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.markdown"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) if err != nil { return err } From aef3b8b31f2e6e508ba66e9eb33b15042633df57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 24 May 2024 13:21:42 +0200 Subject: [PATCH 22/47] Adapt patch to upstream changes Test with different objectFormats. --- tests/integration/git_annex_test.go | 148 ++++++++++++++-------------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 2a016c61dc..db1bb8a498 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -124,98 +124,100 @@ func TestGitAnnexViews(t *testing.T) { } onGiteaRun(t, func(t *testing.T, u *url.URL) { - ctx := NewAPITestContext(t, "user2", "annex-template-render-test", auth_model.AccessTokenScopeWriteRepository) + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-template-render-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) - // create a public repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) - session := loginUser(t, ctx.Username) + session := loginUser(t, ctx.Username) - t.Run("Index", func(t *testing.T) { - // test that annex symlinks renders with the _file icon_ on the main list - defer tests.PrintCurrentTest(t)() + t.Run("Index", func(t *testing.T) { + // test that annex symlinks renders with the _file icon_ on the main list + defer tests.PrintCurrentTest(t)() - repoLink := path.Join("/", ctx.Username, ctx.Reponame) - req := NewRequest(t, "GET", repoLink) - resp := session.MakeRequest(t, req, http.StatusOK) - - htmlDoc := NewHTMLParser(t, resp.Body) - isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") - require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") - }) - - t.Run("View", func(t *testing.T) { - // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files - defer tests.PrintCurrentTest(t)() - - doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { - viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) - // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? - mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) - - req := NewRequest(t, "GET", viewLink) + repoLink := path.Join("/", ctx.Username, ctx.Reponame) + req := NewRequest(t, "GET", repoLink) resp := session.MakeRequest(t, req, http.StatusOK) - htmlDoc = NewHTMLParser(t, resp.Body) - // the first button on the toolbar on the view template is the "Raw" button - // this CSS selector is the most precise I can think to use - buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") - require.True(t, exists, "Download button should exist on the file header") - require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") - - return htmlDoc, viewLink, mediaLink - } - - t.Run("Binary", func(t *testing.T) { - // test that annexing a file renders the /media link in /src and NOT the /raw link - defer tests.PrintCurrentTest(t)() - - doBinaryViewTest := func(file string) { - htmlDoc, _, mediaLink := doViewTest(file) - - rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") - require.True(t, exists, "Download link should render instead of content because this is a binary file") - require.EqualValues(t, mediaLink, rawLink) - } - - t.Run("AnnexSymlink", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doBinaryViewTest("annexed.tiff") - }) - t.Run("AnnexPointer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doBinaryViewTest("annexed.bin") - }) + htmlDoc := NewHTMLParser(t, resp.Body) + isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") + require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") }) - t.Run("Text", func(t *testing.T) { + t.Run("View", func(t *testing.T) { + // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files defer tests.PrintCurrentTest(t)() - doTextViewTest := func(file string) { - htmlDoc, _, _ := doViewTest(file) - require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { + viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) + // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? + mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) + + req := NewRequest(t, "GET", viewLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc = NewHTMLParser(t, resp.Body) + // the first button on the toolbar on the view template is the "Raw" button + // this CSS selector is the most precise I can think to use + buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") + require.True(t, exists, "Download button should exist on the file header") + require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") + + return htmlDoc, viewLink, mediaLink } - t.Run("AnnexSymlink", func(t *testing.T) { + t.Run("Binary", func(t *testing.T) { + // test that annexing a file renders the /media link in /src and NOT the /raw link defer tests.PrintCurrentTest(t)() - doTextViewTest("annexed.txt") - t.Run("Markdown", func(t *testing.T) { - // special case: check that markdown can be pulled out of the annex and rendered, too + doBinaryViewTest := func(file string) { + htmlDoc, _, mediaLink := doViewTest(file) + + rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") + require.True(t, exists, "Download link should render instead of content because this is a binary file") + require.EqualValues(t, mediaLink, rawLink) + } + + t.Run("AnnexSymlink", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - htmlDoc, _, _ := doViewTest("annexed.md") - require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + doBinaryViewTest("annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.bin") }) }) - t.Run("AnnexPointer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doTextViewTest("annexed.rst") - t.Run("Markdown", func(t *testing.T) { - // special case: check that markdown can be pulled out of the annex and rendered, too + t.Run("Text", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + doTextViewTest := func(file string) { + htmlDoc, _, _ := doViewTest(file) + require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + } + + t.Run("AnnexSymlink", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - htmlDoc, _, _ := doViewTest("annexed.markdown") - require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + doTextViewTest("annexed.txt") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.md") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.rst") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.markdown") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) }) }) }) From ad8476c821863c0aef54e0bccaf4c9ce9b244204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 15 May 2024 10:55:12 +0200 Subject: [PATCH 23/47] Add git-annex to docker image --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9a8877920a..70c649679d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ RUN apk --no-cache add \ sqlite \ su-exec \ gnupg \ + git-annex \ && rm -rf /var/cache/apk/* RUN addgroup \ From 4e03aa909580a9e29936dc2c4e2799d7cadc49b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 18 Jul 2024 16:43:42 +0000 Subject: [PATCH 24/47] Error if git-annex is enabled but missing (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copied from https://github.com/neuropoly/gitea/pull/47 This adds a check so that if `setting.Annex.Enabled` is true and git-annex is not in the PATH Forgejo will abort on startup with a reasonable error message. Fixes #15. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/16 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- cmd/web.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/web.go b/cmd/web.go index ecc53698e3..4e89153ab4 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -258,6 +259,12 @@ func runWeb(ctx *cli.Context) error { createPIDFile(ctx.String("pid")) } + if setting.Annex.Enabled { + if _, err := exec.LookPath("git-annex"); err != nil { + log.Fatal("You have enabled git-annex support but git-annex is not installed. Please make sure that Forgejo's PATH contains the git-annex executable.") + } + } + if !setting.InstallLock { if err := serveInstall(ctx); err != nil { return err From 88095d85ea0d3722dd37fb705bf720c3f66397ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 18 Jul 2024 18:18:06 +0000 Subject: [PATCH 25/47] Git-annex web uploads (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements support for uploading files into the annex using the web interface. If a repository is a git-annex-enabled repository all files will be added to it using git annex add. This means that the repository's configuration for what to put into the annex (annex.largefiles in gitattributes) will be respected. Plain git repositories without git-annex will work as before, directly uploading to git. Fixes #5. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/21 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 6 ++ modules/util/remove.go | 11 ++-- services/repository/files/temp_repo.go | 20 +++++++ services/repository/files/upload.go | 83 +++++++++++++++++++++++++- tests/integration/git_annex_test.go | 81 +++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 7 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 96b604693e..01d7743421 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -152,3 +152,9 @@ func IsAnnexed(blob *git.Blob) (bool, error) { } return true, nil } + +// IsAnnexRepo determines if repo is a git-annex enabled repository +func IsAnnexRepo(repo *git.Repository) bool { + _, _, err := git.NewCommand(repo.Ctx, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repo.Path}) + return err == nil +} diff --git a/modules/util/remove.go b/modules/util/remove.go index 4e5271136d..b07a48bee4 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -44,10 +44,13 @@ func MakeWritable(name string) error { return err } - // 0200 == u+w, in octal unix permission notation - err = os.Chmod(path, info.Mode()|0o200) - if err != nil { - return err + // Don't try chmod'ing symlinks (will fail with broken symlinks) + if info.Mode()&os.ModeSymlink != os.ModeSymlink { + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } } } return nil diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index b3aadbc6cb..5840fd6a4f 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -202,6 +202,26 @@ func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPat return nil } +// InitPrivateAnnex initializes a private annex in the repository +func (t *TemporaryUploadRepository) InitPrivateAnnex() error { + if _, _, err := git.NewCommand(t.ctx, "config", "annex.private", "true").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return err + } + if _, _, err := git.NewCommand(t.ctx, "annex", "init").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return err + } + return nil +} + +// AddAnnex adds the file at path to the repository using git annex add +// This requires a non-bare repository +func (t *TemporaryUploadRepository) AddAnnex(path string) error { + if _, _, err := git.NewCommand(t.ctx, "annex", "add").AddDynamicArguments(path).RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return err + } + return nil +} + // WriteTree writes the current index as a tree to the object db and returns its hash func (t *TemporaryUploadRepository) WriteTree() (string, error) { stdout, _, err := git.NewCommand(t.ctx, "write-tree").RunStdString(&git.RunOpts{Dir: t.basePath}) diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go index 6359087e88..39a3f1d74d 100644 --- a/services/repository/files/upload.go +++ b/services/repository/files/upload.go @@ -6,13 +6,16 @@ package files import ( "context" "fmt" + "io" "os" "path" + "path/filepath" "strings" git_model "forgejo.org/models/git" repo_model "forgejo.org/models/repo" user_model "forgejo.org/models/user" + "forgejo.org/modules/annex" "forgejo.org/modules/git" "forgejo.org/modules/lfs" "forgejo.org/modules/setting" @@ -89,7 +92,7 @@ func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use defer t.Close() hasOldBranch := true - if err = t.Clone(opts.OldBranch, true); err != nil { + if err = t.Clone(opts.OldBranch, false); err != nil { if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { return err } @@ -105,10 +108,30 @@ func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } } - // Copy uploaded files into repository. - if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil { + r, err := git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { return err } + if annex.IsAnnexRepo(r) { + // Initialize annex privately in temporary clone + if err := t.InitPrivateAnnex(); err != nil { + return err + } + // Copy uploaded files into git-annex repository + if err := copyUploadedFilesIntoAnnexRepository(infos, t, opts.TreePath); err != nil { + return err + } + // Move all annexed content in the temporary repository, i.e. everything we have just added, to the origin + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + if err := moveAnnexedFilesToOrigin(t, author, committer); err != nil { + return err + } + } else { + // Copy uploaded files into repository. + if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil { + return err + } + } // Now write the tree treeHash, err := t.WriteTree() @@ -246,3 +269,57 @@ func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) er } return nil } + +func copyUploadedFilesIntoAnnexRepository(infos []uploadInfo, t *TemporaryUploadRepository, treePath string) error { + for i := range len(infos) { + if err := copyUploadedFileIntoAnnexRepository(&infos[i], t, treePath); err != nil { + return err + } + } + return nil +} + +func copyUploadedFileIntoAnnexRepository(info *uploadInfo, t *TemporaryUploadRepository, treePath string) error { + pathInRepo := path.Join(t.basePath, treePath, info.upload.Name) + if err := os.MkdirAll(filepath.Dir(pathInRepo), 0o700); err != nil { + return err + } + if err := os.Rename(info.upload.LocalPath(), pathInRepo); err != nil { + // Rename didn't work, try copy and remove + inputFile, err := os.Open(info.upload.LocalPath()) + if err != nil { + return fmt.Errorf("could not open source file: %v", err) + } + defer inputFile.Close() + outputFile, err := os.Create(pathInRepo) + if err != nil { + return fmt.Errorf("could not open dest file: %v", err) + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + if err != nil { + return fmt.Errorf("could not copy to dest from source: %v", err) + } + inputFile.Close() + err = os.Remove(info.upload.LocalPath()) + if err != nil { + return fmt.Errorf("could not remove source file: %v", err) + } + } + return t.AddAnnex(pathInRepo) +} + +func moveAnnexedFilesToOrigin(t *TemporaryUploadRepository, author, committer *user_model.User) error { + authorSig := author.NewGitSig() + committerSig := committer.NewGitSig() + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_COMMITTER_NAME="+committerSig.Name, + "GIT_COMMITTER_EMAIL="+committerSig.Email, + ) + if _, _, err := git.NewCommand(t.ctx, "annex", "move", "--to", "origin").RunStdString(&git.RunOpts{Dir: t.basePath, Env: env}); err != nil { + return err + } + return nil +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index db1bb8a498..b84ab46b2a 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -5,14 +5,17 @@ package integration import ( + "bytes" "errors" "fmt" "io" "math/rand" + "mime/multipart" "net/http" "net/url" "os" "path" + "path/filepath" "regexp" "strings" "testing" @@ -59,6 +62,84 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, return nil } +func TestGitAnnexWebUpload(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-web-upload-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) + + uploadFile := func(t *testing.T, path string) string { + t.Helper() + + body := &bytes.Buffer{} + mpForm := multipart.NewWriter(body) + err := mpForm.WriteField("_csrf", GetCSRF(t, ctx.Session, ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch)) + require.NoError(t, err) + + file, err := mpForm.CreateFormFile("file", filepath.Base(path)) + require.NoError(t, err) + + srcFile, err := os.Open(path) + require.NoError(t, err) + + io.Copy(file, srcFile) + require.NoError(t, mpForm.Close()) + + req := NewRequestWithBody(t, "POST", "/"+ctx.Username+"/"+ctx.Reponame+"/upload-file", body) + req.Header.Add("Content-Type", mpForm.FormDataContentType()) + resp := ctx.Session.MakeRequest(t, req, http.StatusOK) + + respMap := map[string]string{} + DecodeJSON(t, resp, &respMap) + return respMap["uuid"] + } + + // Generate random file + tmpFile := path.Join(t.TempDir(), "web-upload-test-file.bin") + require.NoError(t, generateRandomFile(1024*1024/4, tmpFile)) + expectedContent, err := os.ReadFile(tmpFile) + require.NoError(t, err) + + // Upload generated file + fileUUID := uploadFile(t, tmpFile) + req := NewRequestWithValues(t, "POST", ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch, map[string]string{ + "commit_choice": "direct", + "files": fileUUID, + "_csrf": GetCSRF(t, ctx.Session, ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch), + "commit_mail_id": "-1", + }) + ctx.Session.MakeRequest(t, req, http.StatusSeeOther) + + // Get some handles on the target repository and file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + repo, err := git.OpenRepository(git.DefaultContext, remoteRepoPath) + require.NoError(t, err) + defer repo.Close() + tree, err := repo.GetTree(setting.Repository.DefaultBranch) + require.NoError(t, err) + treeEntry, err := tree.GetTreeEntryByPath(filepath.Base(tmpFile)) + require.NoError(t, err) + blob := treeEntry.Blob() + + // Check that the uploaded file is annexed + isAnnexed, err := annex.IsAnnexed(blob) + require.NoError(t, err) + require.True(t, isAnnexed) + + // Check that the uploaded file has the correct content + annexedFile, err := annex.Content(blob) + require.NoError(t, err) + actualContent, err := io.ReadAll(annexedFile) + require.NoError(t, err) + require.Equal(t, expectedContent, actualContent) + }) + }) +} + func TestGitAnnexMedia(t *testing.T) { if !setting.Annex.Enabled { t.Skip("Skipping since annex support is disabled.") From 688f53d75217142cd2f5860553d63326e259ee3d Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jul 2024 13:15:26 +0000 Subject: [PATCH 26/47] Add git-annex also to the rootless container (#24) Same as https://codeberg.org/matrss/forgejo-aneksajo/commit/89f8aa0bf5134f7c986ea25856fa9ea32a0e0e8c, but for the rootless container. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/24 Reviewed-by: matrss Co-authored-by: Michael Hanke Co-committed-by: Michael Hanke --- Dockerfile.rootless | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.rootless b/Dockerfile.rootless index 47aae2798d..a6819c6cd2 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -73,6 +73,7 @@ RUN apk --no-cache add \ curl \ gnupg \ openssh-client \ + git-annex \ && rm -rf /var/cache/apk/* RUN addgroup \ From 0dc35349e4cdee1107bf43ce01db4d5f07ebe33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 9 Aug 2024 11:51:11 +0000 Subject: [PATCH 27/47] Improve views for annexed but missing files (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, trying to view files that were annexed, but missing, just led to an uninformative error 500. This was rather confusing. With these changes it now shows the pointer target instead of the (missing) content of the file, and also indicates this situation in the "stored with git-annex" message. For semantic correctness views for missing files return a 404 instead of a 200, as they would with the content present. Fixes #7, fixes #13. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/28 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- options/locale/locale_de-DE.ini | 1 + options/locale/locale_en-US.ini | 1 + routers/web/repo/view.go | 54 +++++++++++++++++++++++---------- templates/repo/file_info.tmpl | 2 +- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index 26f24ab6c7..bc012da26a 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -1350,6 +1350,7 @@ video_not_supported_in_browser=Dein Browser unterstützt das HTML5-„video“-T audio_not_supported_in_browser=Dein Browser unterstützt das HTML5-„audio“-Tag nicht. stored_lfs=Gespeichert mit Git LFS stored_annex=Gespeichert mit Git Annex +stored_annex_not_present = hier nicht vorhanden, versuche git annex whereis symbolic_link=Softlink executable_file=Ausführbare Datei commit_graph=Commit-Graph diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5acf932e23..9ec018df5d 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1368,6 +1368,7 @@ video_not_supported_in_browser = Your browser does not support the HTML5 "video" audio_not_supported_in_browser = Your browser does not support the HTML5 "audio" tag. stored_lfs = Stored with Git LFS stored_annex = Stored with Git Annex +stored_annex_not_present = not present here, try using git annex whereis symbolic_link = Symbolic link executable_file = Executable file vendored = Vendored diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 380218ec31..907e02bc3f 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -210,12 +210,13 @@ func localizedExtensions(ext, languageCode string) (localizedExts []string) { } type fileInfo struct { - isTextFile bool - isLFSFile bool - isAnnexFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + isTextFile bool + isLFSFile bool + isAnnexFile bool + isAnnexFilePresent bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { @@ -230,11 +231,22 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, annexContent, err := annex.Content(blob) if err != nil { - // in the case where annex content is missing, what should happen? - // do we render the page with an error message? - // actually that's not a bad idea, there's some sort of error message situation - // TODO: display an error to the user explaining that their data is missing - return nil, nil, nil, err + // If annex.Content returns an error it can mean that the blob does not + // refer to an annexed file or that it is not present here. Since we already + // checked that it is annexed the latter must be the case. So we return the + // content of the blob instead and indicate that the file is indeed annexed, + // but not present here. The template can then communicate the situation. + dataRc, err := blob.DataAsync() + if err != nil { + return nil, nil, nil, err + } + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(dataRc, buf) + buf = buf[:n] + + st := typesniffer.DetectContentType(buf) + return buf, dataRc, &fileInfo{st.IsText(), false, true, false, blob.Size(), nil, st}, nil } stat, err := annexContent.Stat() @@ -248,7 +260,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st := typesniffer.DetectContentType(buf) - return buf, annexContent, &fileInfo{st.IsText(), false, true, stat.Size(), nil, st}, nil + return buf, annexContent, &fileInfo{st.IsText(), false, true, true, stat.Size(), nil, st}, nil } dataRc, err := blob.DataAsync() @@ -265,18 +277,18 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, // FIXME: what happens when README file is an image? if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, false, blob.Size(), nil, st}, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, false, blob.Size(), nil, st}, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) if err != nil { // fallback to plain file log.Warn("Unable to access LFS pointer %s in repo %d: %v", pointer.Oid, repoID, err) - return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, false, blob.Size(), nil, st}, nil } dataRc.Close() @@ -296,7 +308,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st = typesniffer.DetectContentType(buf) - return buf, dataRc, &fileInfo{st.IsText(), true, false, meta.Size, &meta.Pointer, st}, nil + return buf, dataRc, &fileInfo{st.IsText(), true, false, false, meta.Size, &meta.Pointer, st}, nil } func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.TreeEntry) { @@ -504,6 +516,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { } ctx.Data["IsLFSFile"] = fInfo.isLFSFile ctx.Data["IsAnnexFile"] = fInfo.isAnnexFile + ctx.Data["IsAnnexFilePresent"] = fInfo.isAnnexFilePresent ctx.Data["FileSize"] = fInfo.fileSize ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText @@ -1215,6 +1228,15 @@ PostRecentBranchCheck: } else { ctx.Data["CodeSearchOptions"] = git.GrepSearchOptions } + isAnnexFile, okAnnexFile := ctx.Data["IsAnnexFile"] + isAnnexFilePresent, okAnnexFilePresent := ctx.Data["IsAnnexFilePresent"] + if okAnnexFile && okAnnexFilePresent && isAnnexFile.(bool) && !isAnnexFilePresent.(bool) { + // If the file to be viewed is annexed but not present then render it normally + // (which will show the plain git blob content, i.e. the symlink or pointer target) + // but make the status code a 404. + ctx.HTML(http.StatusNotFound, tplRepoHome) + return + } ctx.HTML(http.StatusOK, tplRepoHome) } diff --git a/templates/repo/file_info.tmpl b/templates/repo/file_info.tmpl index 05d9825cfb..8655404394 100644 --- a/templates/repo/file_info.tmpl +++ b/templates/repo/file_info.tmpl @@ -17,7 +17,7 @@ {{if .FileSize}}
{{ctx.Locale.TrSize .FileSize}}{{if .IsLFSFile}} ({{ctx.Locale.Tr "repo.stored_lfs"}}){{end}} - {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}){{end}} + {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}{{if not .IsAnnexFilePresent}} - {{ctx.Locale.Tr "repo.stored_annex_not_present"}}{{end}}){{end}}
{{end}} {{if .LFSLock}} From 8b50a576ec3fa25be5e12d5a47e6909701c5ff80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 9 Aug 2024 17:17:02 +0000 Subject: [PATCH 28/47] Change the icon for annexed files to file-binary (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #26. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/29 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/base/tool.go | 12 ++++++------ tests/integration/git_annex_test.go | 8 +++++--- web_src/css/repo.css | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index e70d7d4df7..38201c5919 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -102,14 +102,14 @@ func Int64sToStrings(ints []int64) []string { // EntryIcon returns the octicon class for displaying files/directories func EntryIcon(entry *git.TreeEntry) string { + isAnnexed, _ := annex.IsAnnexed(entry.Blob()) + if isAnnexed { + // Show git-annex files as binary files to differentiate them from non-annexed files + // TODO: find a more suitable icon, maybe something related to git-annex + return "file-binary" + } switch { case entry.IsLink(): - isAnnexed, _ := annex.IsAnnexed(entry.Blob()) - if isAnnexed { - // git-annex files are sometimes stored as symlinks; - // short-circuit that so like LFS they are displayed as regular files - return "file" - } te, _, err := entry.FollowLink() if err != nil { log.Debug(err.Error()) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index b84ab46b2a..90bdf0e258 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -214,7 +214,7 @@ func TestGitAnnexViews(t *testing.T) { session := loginUser(t, ctx.Username) t.Run("Index", func(t *testing.T) { - // test that annex symlinks renders with the _file icon_ on the main list + // test that annexed files render with the binary file icon on the main list defer tests.PrintCurrentTest(t)() repoLink := path.Join("/", ctx.Username, ctx.Reponame) @@ -222,8 +222,10 @@ func TestGitAnnexViews(t *testing.T) { resp := session.MakeRequest(t, req, http.StatusOK) htmlDoc := NewHTMLParser(t, resp.Body) - isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") - require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") + isFileBinaryIconLocked := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file-binary") + require.True(t, isFileBinaryIconLocked, "locked annexed files should render a binary file icon") + isFileBinaryIconUnlocked := htmlDoc.Find("tr[data-entryname='annexed.bin'] > td.name svg").HasClass("octicon-file-binary") + require.True(t, isFileBinaryIconUnlocked, "unlocked annexed files should render a binary file icon") }) t.Run("View", func(t *testing.T) { diff --git a/web_src/css/repo.css b/web_src/css/repo.css index f498a992ed..c8049abe61 100644 --- a/web_src/css/repo.css +++ b/web_src/css/repo.css @@ -246,6 +246,7 @@ td .commit-summary { } .repository.file.list #repo-files-table tbody .svg.octicon-file, +.repository.file.list #repo-files-table tbody .svg.octicon-file-binary, .repository.file.list #repo-files-table tbody .svg.octicon-file-symlink-file, .repository.file.list #repo-files-table tbody .svg.octicon-file-directory-symlink { color: var(--color-secondary-dark-7); From 71338f38640ca4ee6b5cc948145620e47120f3d8 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Wed, 4 Sep 2024 09:32:40 +0000 Subject: [PATCH 29/47] Elevate external markup renderer interface for annexed file content (#36) Previously, an external renderer that matched on an annexed file would only see its content streamed via `STDIN`, or a temporary file with a copy of its content would be generated and passed-by-filepath (with `IS_INPUT_FILE=true`). Whether that happens, is also subject to `MAX_DISPLAY_FILE_SIZE` (which defaults to 8MB). This was problematic, because annexed files tend to be large. Moreover, if present, they already exist as write-protected files on the file-system. Creating a copy is both expensive and serves no particular purpose. This commit changes how external renderers are called. 1) With `IS_INPUT_FILE=true`, the renderer is passed the true location of an annex key, if present, and an empty path, if not. 2) The original, repository-relative path of the rendering target is made available to the renderer via the `GITEA_RELATIVE_PATH` environment variable. To achieve a lean implementation, the `Blob` of the rendering target is passed on to the `RenderContext` (because the implementation of the annex-related functionality is centered on this dtype. This change makes it less costly to increase `MAX_DISPLAY_FILE_SIZE`, in order to make large, annexed files eligible for markup rendering, because no content copies will be made any longer. External renderers can now use the original file path, with the full original filename, including extensions, for decision making. For example, to detect particular compression formats based in a file name extension, or to alter the rendering based on contextual information encoded in the file path (e.g., a multi-file data structure with a particular organization pattern). Apart from the additional environment variable, there is no change to the handling of renderers that take their input via `STDIN` (i.e., `IS_INPUT_FILE=false`). Fixes #35. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/36 Reviewed-by: matrss Co-authored-by: Michael Hanke Co-committed-by: Michael Hanke --- modules/markup/external/external.go | 25 +++++++++++++++++++++++-- modules/markup/renderer.go | 20 ++++++++++++-------- routers/web/repo/view.go | 3 +++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 87c1a284cc..950da6e828 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -11,6 +11,7 @@ import ( "os/exec" "strings" + "forgejo.org/modules/annex" "forgejo.org/modules/graceful" "forgejo.org/modules/log" "forgejo.org/modules/markup" @@ -82,8 +83,22 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. commands = strings.Fields(command) args = commands[1:] ) - - if p.IsInputFile { + isAnnexed, _ := annex.IsAnnexed(ctx.Blob) + // if a renderer wants to read a file, and we have annexed content, we can + // provide the annex key file location directly to the renderer. git-annex + // takes care of having that location be read-only, so no critical + // protection layer is needed. Moreover, the file readily exists, and + // expensive temporary files can be avoided, also allowing an operator + // to raise MAX_DISPLAY_FILE_SIZE without much negative impact. + if p.IsInputFile && isAnnexed { + // look for annexed content, will be empty, if there is none + annexContentLocation, _ := annex.ContentLocation(ctx.Blob) + // we call the renderer, even if there is no annex content present. + // showing the pointer file content is not much use, and a topical + // renderer might be able to produce something useful from the + // filename alone (present in ENV) + args = append(args, annexContentLocation) + } else if p.IsInputFile { // write to temp file f, err := os.CreateTemp("", "gitea_input") if err != nil { @@ -126,6 +141,12 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. os.Environ(), "GITEA_PREFIX_SRC="+ctx.Links.SrcLink(), "GITEA_PREFIX_RAW="+ctx.Links.RawLink(), + // also communicate the relative path of the to-be-rendered item. + // this enables the renderer to make use of the original file name + // and path, e.g., to make rendering or dtype-detection decisions + // that go beyond the originally matched extension. Even if the + // content is directly streamed to STDIN + "GITEA_RELATIVE_PATH="+ctx.RelativePath, ) if !p.IsInputFile { cmd.Stdin = input diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index a622d75085..8eec764bbe 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -67,14 +67,18 @@ type Header struct { // RenderContext represents a render context type RenderContext struct { - Ctx context.Context - RelativePath string // relative path from tree root of the branch - Type string - IsWiki bool - Links Links - Metas map[string]string - DefaultLink string - GitRepo *git.Repository + Ctx context.Context + RelativePath string // relative path from tree root of the branch + Type string + IsWiki bool + Links Links + Metas map[string]string + DefaultLink string + GitRepo *git.Repository + // reporting the target blob that is to-be-rendered enables + // deeper inspection in the handler for external renderer + // (i.e., more targeted handling of annexed files) + Blob *git.Blob ShaExistCache map[string]bool cancelFn func() SidebarTocNode ast.Node diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 907e02bc3f..c231b7ee6c 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -371,6 +371,7 @@ func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.Tr }, Metas: ctx.Repo.Repository.ComposeDocumentMetas(ctx), GitRepo: ctx.Repo.GitRepo, + Blob: target.Blob(), }, rd) if err != nil { log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.Name(), ctx.Repo.Repository, err) @@ -607,6 +608,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { }, Metas: metas, GitRepo: ctx.Repo.GitRepo, + Blob: entry.Blob(), }, rd) if err != nil { ctx.ServerError("Render", err) @@ -705,6 +707,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { }, Metas: ctx.Repo.Repository.ComposeDocumentMetas(ctx), GitRepo: ctx.Repo.GitRepo, + Blob: entry.Blob(), }, rd) if err != nil { ctx.ServerError("Render", err) From b41c692e04138430747b4f3c42dbd93497d4e9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 9 Oct 2024 14:44:18 +0000 Subject: [PATCH 30/47] Use PATH when looking for git commands (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes some issues when using a git-annex that is installed in a different location than where git is installed, e.g. when using the git-annex-standalone release or one installed with nix. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/44 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- cmd/serv.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 62e03dfd3b..f9e3bb8d4f 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -339,9 +339,8 @@ func runServ(c *cli.Context) error { return nil } - gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin - gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack - if _, err := os.Stat(gitBinVerb); err != nil { + gitBinVerb, err := exec.LookPath(verb) + if err != nil { // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath, From 2aef6a880d963779bd226d1e757a7c4272523dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 9 Oct 2024 19:33:05 +0000 Subject: [PATCH 31/47] Only upload to annex in doAnnexUploadTest (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation both uploaded to the annex and pushed to the git repository. This meant that the tests checking that uploads without permission fail actually could pass when the git push failed but the git-annex upload didn't. The tests didn't catch the situation where unauthorized users could modify the annex. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/46 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 23 +++++----- tests/integration/git_annex_test.go | 68 ++++++++++++++++------------- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 01d7743421..e2101366eb 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -104,6 +104,18 @@ func Pointer(blob *git.Blob) (string, error) { return pointer, nil } +func ContentLocationFromPointer(repoPath, pointer string) (string, error) { + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", repoPath, pointer, err) + } + contentLocation = strings.TrimSpace(contentLocation) + contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals + contentLocation = path.Join(repoPath, contentLocation) + + return contentLocation, nil +} + // return the absolute path of the content pointed to by the annex pointer stored in the git object // errors if the content is not found in this repo func ContentLocation(blob *git.Blob) (string, error) { @@ -111,16 +123,7 @@ func ContentLocation(blob *git.Blob) (string, error) { if err != nil { return "", err } - - contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) - if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err) - } - contentLocation = strings.TrimSpace(contentLocation) - contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals - contentLocation = path.Join(blob.Repo().Path, contentLocation) - - return contentLocation, nil + return ContentLocationFromPointer(blob.Repo().Path, pointer) } // returns a stream open to the annex content diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 90bdf0e258..886b6dba63 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -1119,19 +1119,21 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - return err - } - // verify the file was uploaded - localObjectPath, err := contentLocation(repoPath, "contribution.bin") + blob, err := blobForFile(repoPath, "contribution.bin") + if err != nil { + return err + } + key, err := annex.Pointer(blob) + if err != nil { + return err + } + localObjectPath, err := annex.ContentLocationFromPointer(repoPath, key) if err != nil { return err } - // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file - remoteObjectPath, err := contentLocation(remoteRepoPath, "contribution.bin") + remoteObjectPath, err := annex.ContentLocationFromPointer(remoteRepoPath, key) if err != nil { return err } @@ -1325,6 +1327,31 @@ func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { return nil } +func blobForFile(repoPath, file string) (*git.Blob, error) { + repo, err := git.OpenRepository(git.DefaultContext, repoPath) + if err != nil { + return nil, err + } + defer repo.Close() + + commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags + if err != nil { + return nil, err + } + + commit, err := repo.GetCommit(commitID) + if err != nil { + return nil, err + } + + treeEntry, err := commit.GetTreeEntryByPath(file) + if err != nil { + return nil, err + } + + return treeEntry.Blob(), nil +} + /* Find the path in .git/annex/objects/ of the contents for a given annexed file. @@ -1334,30 +1361,11 @@ Find the path in .git/annex/objects/ of the contents for a given annexed file. TODO: pass a parameter to allow examining non-HEAD branches */ func contentLocation(repoPath, file string) (path string, err error) { - path = "" - - repo, err := git.OpenRepository(git.DefaultContext, repoPath) + blob, err := blobForFile(repoPath, file) if err != nil { - return path, nil + return "", err } - defer repo.Close() - - commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags - if err != nil { - return path, nil - } - - commit, err := repo.GetCommit(commitID) - if err != nil { - return path, nil - } - - treeEntry, err := commit.GetTreeEntryByPath(file) - if err != nil { - return path, nil - } - - return annex.ContentLocation(treeEntry.Blob()) + return annex.ContentLocation(blob) } /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ From be55d4233a15ca0a0cd68bfd7fdb3191c521a9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 14 Oct 2024 15:59:21 +0000 Subject: [PATCH 32/47] Add git-annex' testremote to the test suite (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `git annex testremote` command runs a built-in set of tests against a remote. It cannot hurt to check our implementation of a git-annex remote against it too. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/48 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- tests/integration/git_annex_test.go | 196 ++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 886b6dba63..c9c5d3cabf 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -395,6 +395,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -425,6 +435,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -457,6 +477,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -487,6 +517,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -519,6 +559,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -549,6 +599,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -581,6 +641,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -611,6 +681,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -642,6 +722,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) @@ -713,6 +803,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -743,6 +843,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -775,6 +885,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -805,6 +925,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -837,6 +967,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -867,6 +1007,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -899,6 +1049,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -929,6 +1089,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -960,6 +1130,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) @@ -1032,6 +1212,22 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { return nil } +func doAnnexTestremoteReadWriteTest(repoPath string) (err error) { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "testremote", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + return nil +} + +func doAnnexTestremoteReadOnlyTest(repoPath string) (err error) { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "testremote", "origin", "--test-readonly", "annexed.tiff").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + return nil +} + func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { // NB: this test does something slightly different if run separately from "doAnnexInitTest()": // "git annex copy" will notice and run "git annex init", silently. From 59510cd90d5d6aab3d94857c44ef0ac0a5a4e701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 22 Oct 2024 18:33:26 +0000 Subject: [PATCH 33/47] Add tests for git annex drop (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds some rudimentary tests that drop files in a repository's clone as well as from a repository on Forgejo. Fixes #4. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/47 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- tests/integration/git_annex_test.go | 1004 +++++++++++++++++++++------ 1 file changed, 796 insertions(+), 208 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index c9c5d3cabf..1c12f3a032 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -382,34 +382,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -425,24 +451,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -464,34 +524,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -507,24 +593,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -546,34 +666,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") - }) }) }) @@ -589,24 +735,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -628,34 +808,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") - }) }) }) @@ -671,24 +877,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -723,6 +963,26 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + t.Run("TestremoteReadOnly", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) @@ -790,34 +1050,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -833,24 +1119,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -872,34 +1192,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -915,24 +1261,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -954,34 +1334,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") - }) }) }) @@ -997,24 +1403,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -1036,34 +1476,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") - }) }) }) @@ -1079,24 +1545,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -1131,6 +1631,26 @@ func TestGitAnnexPermissions(t *testing.T) { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + t.Run("TestremoteReadOnly", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) @@ -1290,6 +1810,40 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } +func doAnnexLocalDropTest(repoPath string) (err error) { + // This test assumes that files are present in repoPath, i.e. it is run after doAnnexDownloadTest. + // This test drops all files from the repository clone. + binPath, err := contentLocation(repoPath, "annexed.bin") + if err != nil { + return err + } + _, err = os.Stat(binPath) + if err != nil { + return err + } + tiffPath, err := contentLocation(repoPath, "annexed.tiff") + if err != nil { + return err + } + _, err = os.Stat(tiffPath) + if err != nil { + return err + } + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "drop").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + _, err = os.Stat(binPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.bin wasn't dropped properly: %w", err) + } + _, err = os.Stat(tiffPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.tiff wasn't dropped properly: %w", err) + } + return nil +} + func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { // NB: this test does something slightly different if run separately from "Init": // it first runs "git annex init" silently in the background. @@ -1345,6 +1899,40 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return nil } +func doAnnexRemoteDropTest(remoteRepoPath, repoPath string) (err error) { + // This test assumes that files are present in repoPath, i.e. it is run after doAnnexDownloadTest. + // This test drops all files from the remote repository. + binPath, err := contentLocation(remoteRepoPath, "annexed.bin") + if err != nil { + return err + } + _, err = os.Stat(binPath) + if err != nil { + return err + } + tiffPath, err := contentLocation(remoteRepoPath, "annexed.tiff") + if err != nil { + return err + } + _, err = os.Stat(tiffPath) + if err != nil { + return err + } + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "drop", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + _, err = os.Stat(binPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.bin wasn't dropped properly: %w", err) + } + _, err = os.Stat(tiffPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.tiff wasn't dropped properly: %w", err) + } + return nil +} + // ---- Helpers ---- func generateRandomFile(size int, path string) (err error) { From eba1a9626375494f5232ffdb84d284d21ab91a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 25 Oct 2024 09:55:56 +0000 Subject: [PATCH 34/47] Add git-annex p2phttp support (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new endpoint under `/git-annex-p2phttp` which acts as an authenticating proxy to git-annex' p2phttp server. This makes it possible to set `annex+/git-annex-p2phttp` as `remote..annexurl` and use git-annex fully over http(s) with the normal credentials and access tokens provided by Forgejo. Fixes #25. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/42 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- .forgejo/workflows/testing.yml | 3 - custom/conf/app.example.ini | 2 + modules/annex/annex.go | 28 + modules/setting/annex.go | 7 +- routers/web/repo/annex.go | 146 +++++ routers/web/repo/githttp.go | 20 + routers/web/web.go | 12 +- services/auth/auth.go | 2 +- tests/integration/git_annex_test.go | 888 +++++++++++++++++++++++++--- 9 files changed, 1012 insertions(+), 96 deletions(-) create mode 100644 routers/web/repo/annex.go diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 7e3a951872..713dfebf63 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -200,7 +200,6 @@ jobs: - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-mysql-migration test-mysql' - timeout-minutes: 120 env: USE_REPO_TEST_DIR: 1 test-pgsql: @@ -236,7 +235,6 @@ jobs: - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-pgsql-migration test-pgsql' - timeout-minutes: 120 env: RACE_ENABLED: true USE_REPO_TEST_DIR: 1 @@ -257,7 +255,6 @@ jobs: - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-sqlite-migration test-sqlite' - timeout-minutes: 120 env: TAGS: sqlite sqlite_unlock_notify RACE_ENABLED: true diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 789178d9be..b76cf7df80 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2686,6 +2686,8 @@ LEVEL = Info ;; ;; Whether git-annex is enabled; defaults to false ;ENABLED = false +;; Whether to disable p2phttp support; default is the same as repository.DISABLE_HTTP_GIT +;DISABLE_P2PHTTP = false ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/modules/annex/annex.go b/modules/annex/annex.go index e2101366eb..016f8ca8fb 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -12,8 +12,11 @@ package annex import ( "errors" "fmt" + "io/fs" "os" "path" + "path/filepath" + "regexp" "strings" "forgejo.org/modules/git" @@ -161,3 +164,28 @@ func IsAnnexRepo(repo *git.Repository) bool { _, _, err := git.NewCommand(repo.Ctx, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repo.Path}) return err == nil } + +var repoConfigFileRe = regexp.MustCompile("[^/]+/[^/]+.git/config$") + +func UUID2RepoPath(uuid string) (string, error) { + var repoPath string + err := filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { + if err == nil && repoConfigFileRe.MatchString(path) { + thisRepoPath := strings.TrimSuffix(path, "/config") + stdout, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: thisRepoPath}) + if err != nil { + return nil + } + repoUUID := strings.TrimSpace(stdout) + if repoUUID == uuid { + repoPath = thisRepoPath + return fs.SkipAll + } + } + return nil + }) + if err != nil { + return "", err + } + return repoPath, nil +} diff --git a/modules/setting/annex.go b/modules/setting/annex.go index b8616ba87d..aa41c14ff0 100644 --- a/modules/setting/annex.go +++ b/modules/setting/annex.go @@ -9,7 +9,8 @@ import ( // Annex represents the configuration for git-annex var Annex = struct { - Enabled bool `ini:"ENABLED"` + Enabled bool `ini:"ENABLED"` + DisableP2PHTTP bool `ini:"DISABLE_P2PHTTP"` }{} func loadAnnexFrom(rootCfg ConfigProvider) { @@ -17,4 +18,8 @@ func loadAnnexFrom(rootCfg ConfigProvider) { if err := sec.MapTo(&Annex); err != nil { log.Fatal("Failed to map Annex settings: %v", err) } + if !sec.HasKey("DISABLE_P2PHTTP") { + // If DisableP2PHTTP is not explicitly set then use DisableHTTPGit as its default + Annex.DisableP2PHTTP = Repository.DisableHTTPGit + } } diff --git a/routers/web/repo/annex.go b/routers/web/repo/annex.go new file mode 100644 index 0000000000..facdded335 --- /dev/null +++ b/routers/web/repo/annex.go @@ -0,0 +1,146 @@ +package repo + +import ( + "context" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "forgejo.org/models/perm" + access_model "forgejo.org/models/perm/access" + repo_model "forgejo.org/models/repo" + "forgejo.org/models/unit" + "forgejo.org/modules/annex" + "forgejo.org/modules/graceful" + "forgejo.org/modules/log" + services_context "forgejo.org/services/context" +) + +type p2phttpRecordType struct { + CancelFunc func() + LastUsed time.Time + Port string +} + +var p2phttpRecords = make(map[string]*p2phttpRecordType) + +// AnnexP2PHTTP implements git-annex smart HTTP support by delegating to git annex p2phttp +func AnnexP2PHTTP(ctx *services_context.Context) { + uuid := ctx.Params(":uuid") + repoPath, err := annex.UUID2RepoPath(uuid) + if err != nil { + ctx.PlainText(http.StatusNotFound, "Repository not found") + return + } + + parts := strings.Split(repoPath, "/") + repoName := strings.TrimSuffix(parts[len(parts)-1], ".git") + owner := parts[len(parts)-2] + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner, repoName) + if err != nil { + ctx.PlainText(http.StatusNotFound, "Repository not found") + return + } + + p, err := access_model.GetUserRepoPermission(ctx, repo, ctx.Doer) + if err != nil { + ctx.ServerError("GetUserRepoPermission", err) + return + } + + if !(ctx.Req.Method == "GET" && p.CanAccess(perm.AccessModeRead, unit.TypeCode) || + ctx.Req.Method == "POST" && p.CanAccess(perm.AccessModeWrite, unit.TypeCode) || + ctx.Req.Method == "POST" && strings.HasSuffix(ctx.Req.URL.Path, "/checkpresent") && p.CanAccess(perm.AccessModeRead, unit.TypeCode) || + ctx.Req.Method == "POST" && strings.HasSuffix(ctx.Req.URL.Path, "/keeplocked") || + ctx.Req.Method == "POST" && strings.HasSuffix(ctx.Req.URL.Path, "/lockcontent")) { + // GET requests require at least read access; POST requests for + // anything but checkpresent, lockcontent, and keeplocked + // require write permissions; POST requests for checkpresent + // only require read permissions, as it really is just a read. + // POST requests for lockcontent and keeplocked require no + // authentication at all, as is also the case for the + // authentication in the git-annex-p2phttp server. See + // https://git-annex.branchable.com/bugs/p2phttp__58___drop_difference_wideopen_unauth-readonly/ + // for reasoning. + ctx.Resp.WriteHeader(http.StatusUnauthorized) + return + } + + p2phttpRecord, p2phttpProcessExists := p2phttpRecords[uuid] + if p2phttpProcessExists { + p2phttpRecord.LastUsed = time.Now() + } else { + // Start a new p2phttp process for the requested repository + // There is a race condition here with the port selection, ideally git annex p2phttp could just listen on a unix socket... + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Error("Failed to listen on a free port: %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + hopefullyFreePort := strings.SplitN(lis.Addr().String(), ":", 2)[1] + lis.Close() + p2phttpCtx, p2phttpCtxCancel := context.WithCancel(context.Background()) + go func(ctx context.Context) { + cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "annex", "p2phttp", "-J2", "--bind", "127.0.0.1", "--wideopen", "--port", hopefullyFreePort) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGINT, + } + cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } + _ = cmd.Run() + }(p2phttpCtx) + graceful.GetManager().RunAtTerminate(p2phttpCtxCancel) + + // Wait for the p2phttp server to get ready + start := time.Now() + sleepDuration := 1 * time.Millisecond + for { + if time.Since(start) > 5*time.Second { + p2phttpCtxCancel() + log.Error("Failed to start the p2phttp server in a reasonable amount of time") + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + conn, err := net.Dial("tcp", "127.0.0.1:"+hopefullyFreePort) + if err == nil { + conn.Close() + break + } + time.Sleep(sleepDuration) + sleepDuration *= 2 + if sleepDuration > 1*time.Second { + sleepDuration = 1 * time.Second + } + } + + p2phttpRecord = &p2phttpRecordType{CancelFunc: p2phttpCtxCancel, LastUsed: time.Now(), Port: hopefullyFreePort} + p2phttpRecords[uuid] = p2phttpRecord + } + + // Cleanup p2phttp processes that haven't been used for a while + for uuid, record := range p2phttpRecords { + if time.Since(record.LastUsed) > 5*time.Minute { + record.CancelFunc() + delete(p2phttpRecords, uuid) + } + } + + url, err := url.Parse("http://127.0.0.1:" + p2phttpRecord.Port + strings.TrimPrefix(ctx.Req.RequestURI, "/git-annex-p2phttp")) + if err != nil { + log.Error("Failed to parse URL: %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + proxy := httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + r.Out.URL = url + }, + } + proxy.ServeHTTP(ctx.Resp, ctx.Req) +} diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index 3c95f4bd4c..3e80e120b5 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -545,6 +545,26 @@ func GetInfoRefs(ctx *context.Context) { } } +// GetConfig implements fetching the git config of a repository +func GetConfig(ctx *context.Context) { + h := httpBase(ctx) + if h != nil { + setHeaderNoCache(ctx) + config, err := os.ReadFile(filepath.Join(h.getRepoDir(), "config")) + if err != nil { + log.Error("Failed to read git config file: %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + if !setting.Annex.DisableP2PHTTP { + config = append(config, []byte("[annex]\n\turl = annex+"+setting.AppURL+"git-annex-p2phttp\n")...) + } + ctx.Resp.Header().Set("Content-Type", "text/plain") + ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(config))) + http.ServeContent(ctx.Resp, ctx.Req, "config", time.Now(), bytes.NewReader(config)) + } +} + // GetTextFile implements Git dumb HTTP func GetTextFile(p string) func(*context.Context) { return func(ctx *context.Context) { diff --git a/routers/web/web.go b/routers/web/web.go index 91400d5d17..2840ee419a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -357,6 +357,13 @@ func registerRoutes(m *web.Route) { } } + annexP2PHTTPEnabled := func(ctx *context.Context) { + if setting.Annex.DisableP2PHTTP { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -959,6 +966,9 @@ func registerRoutes(m *web.Route) { // ***** END: Organization ***** // ***** START: Repository ***** + m.Group("", func() { + m.Methods("GET,POST", "/git-annex-p2phttp/git-annex/{uuid}/*", repo.AnnexP2PHTTP) + }, ignSignInAndCsrf, annexEnabled, annexP2PHTTPEnabled) m.Group("/repo", func() { m.Get("/create", repo.Create) m.Post("/create", web.Bind(forms.CreateRepoForm{}), repo.CreatePost) @@ -1641,7 +1651,7 @@ func registerRoutes(m *web.Route) { m.Group("", func() { // for git-annex - m.Methods("GET,OPTIONS", "/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.Methods("GET,OPTIONS", "/config", repo.GetConfig) // needed by clients reading annex.uuid during `git annex initremote` m.Methods("GET,OPTIONS", "/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) }, ignSignInAndCsrf, annexEnabled, context.UserAssignmentWeb()) diff --git a/services/auth/auth.go b/services/auth/auth.go index 9d72a88c1f..d0fa725854 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -61,7 +61,7 @@ func isArchivePath(req *http.Request) bool { return archivePathRe.MatchString(req.URL.Path) } -var annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) +var annexPathRe = regexp.MustCompile(`^(/git-annex-p2phttp/|/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/)`) func isAnnexPath(req *http.Request) bool { if setting.Annex.Enabled { diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 1c12f3a032..d09dc558a2 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -458,6 +458,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { @@ -507,6 +511,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -600,6 +673,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, writerCtx, func() { @@ -649,6 +726,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -742,6 +888,79 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, readerCtx, func() { @@ -884,6 +1103,79 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err = git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { @@ -938,7 +1230,7 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("Anonymous", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - // Only HTTP has an anonymous mode + // Only HTTP and P2PHTTP have an anonymous mode t.Run("HTTP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -958,6 +1250,65 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) @@ -1126,6 +1477,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { @@ -1175,6 +1530,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -1268,6 +1692,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, writerCtx, func() { @@ -1317,6 +1745,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -1410,6 +1907,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, readerCtx, func() { @@ -1459,81 +1960,8 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) - }) - t.Run("Outsider", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - repoURL := createSSHUrl(ownerCtx.GitPath(), u) - - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) - - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") - }) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") - }) - }) - - t.Run("LocalDrop", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexLocalDropTest(repoPath)) - }) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) - }) - - t.Run("RemoteDrop", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) - }) - }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") - }) - }) - - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) - }) - }) - - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) - }) - }) - }) - - t.Run("HTTP", func(t *testing.T) { + t.Run("P2PHTTP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() repoURL := createHTTPUrl(ownerCtx.GitPath(), u) @@ -1547,66 +1975,281 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("Init", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) }) t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) }) t.Run("LocalDrop", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexLocalDropTest(repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) }) }) t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) }) t.Run("RemoteDrop", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) }) }) t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) }) }) t.Run("TestremoteReadOnly", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) }) t.Run("TestremoteReadWrite", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + // Try unsetting annexurl + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + }) }) t.Run("Anonymous", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - // Only HTTP has an anonymous mode + // Only HTTP and P2PHTTP have an anonymous mode t.Run("HTTP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -1626,6 +2269,65 @@ func TestGitAnnexPermissions(t *testing.T) { require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + // Try unsetting annexurl + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) @@ -2189,8 +2891,14 @@ func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, call credentialedURL := *u credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + credentialedAnnexURL := *u + credentialedAnnexURL.Host = strings.ReplaceAll(credentialedAnnexURL.Host, "127.0.0.1", "localhost") + credentialedAnnexURL.Scheme = "annex+" + credentialedAnnexURL.Scheme + credentialedAnnexURL.Path += "git-annex-p2phttp" + credentialedAnnexURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + creds := path.Join(t.TempDir(), "creds") - require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()+"\n"+credentialedAnnexURL.String()+"\n"), 0o600)) originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) if err != nil && !git.IsErrorExitCode(err, 1) { From 14c0633698b03a045c669b6f5bbd0f50bfa0762b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 25 Oct 2024 09:56:36 +0000 Subject: [PATCH 35/47] Simplify git blob to annex key lookup (#43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #27. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/43 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 102 ++++------------------------ tests/integration/git_annex_test.go | 6 +- 2 files changed, 18 insertions(+), 90 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 016f8ca8fb..5d239f7752 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -21,96 +21,24 @@ import ( "forgejo.org/modules/git" "forgejo.org/modules/setting" - "forgejo.org/modules/util" ) -const ( - // > The maximum size of a pointer file is 32 kb. - // - https://git-annex.branchable.com/internals/pointer_file/ - // It's unclear if that's kilobytes or kibibytes; assuming kibibytes: - blobSizeCutoff = 32 * 1024 -) +// ErrBlobIsNotAnnexed occurs if a blob does not contain a valid annex key +var ErrBlobIsNotAnnexed = errors.New("not a git-annex pointer") -// ErrInvalidPointer occurs if the pointer's value doesn't parse -var ErrInvalidPointer = errors.New("Not a git-annex pointer") - -// Gets the content of the blob as raw text, up to n bytes. -// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit) -func getBlobContent(b *git.Blob, n int) (string, error) { - dataRc, err := b.DataAsync() +func LookupKey(blob *git.Blob) (string, error) { + stdout, _, err := git.NewCommand(git.DefaultContext, "annex", "lookupkey", "--ref").AddDynamicArguments(blob.ID.String()).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) if err != nil { - return "", err + return "", ErrBlobIsNotAnnexed } - defer dataRc.Close() - buf := make([]byte, n) - n, _ = util.ReadAtMost(dataRc, buf) - buf = buf[:n] - return string(buf), nil + key := strings.TrimSpace(stdout) + return key, nil } -func Pointer(blob *git.Blob) (string, error) { - // git-annex doesn't seem fully spec what its pointer are, but - // the fullest description is here: - // https://git-annex.branchable.com/internals/pointer_file/ - - // a pointer can be: - // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY' - // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY' - // - // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because - // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one. - - if blob.Size() > blobSizeCutoff { - // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file. - // https://git-annex.branchable.com/internals/pointer_file/ - - // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too. - return "", ErrInvalidPointer - } - - pointer, err := getBlobContent(blob, blobSizeCutoff) +func ContentLocationFromKey(repoPath, key string) (string, error) { + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(key).RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { - return "", fmt.Errorf("error reading %s: %w", blob.Name(), err) - } - - // the spec says a pointer file can contain multiple lines each with a pointer in them - // but that makes no sense to me, so I'm just ignoring all but the first - lines := strings.Split(pointer, "\n") - if len(lines) < 1 { - return "", ErrInvalidPointer - } - pointer = lines[0] - - // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it - if !strings.Contains(pointer, "/annex/") { - return "", ErrInvalidPointer - } - - // extract $KEY - pointer = path.Base(strings.TrimSpace(pointer)) - - // ask git-annex's opinion on $KEY - // XXX: this is probably a bit slow, especially if this operation gets run often - // and examinekey is not that strict: - // - it doesn't enforce that the "BACKEND" tag is one it knows, - // - it doesn't enforce that the fields and their format fit the "BACKEND" tag - // so maybe this is a wasteful step - _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) - if err != nil { - // TODO: make ErrInvalidPointer into a type capable of wrapping err - if strings.TrimSpace(examineStderr) == "git-annex: bad key" { - return "", ErrInvalidPointer - } - return "", err - } - - return pointer, nil -} - -func ContentLocationFromPointer(repoPath, pointer string) (string, error) { - contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", repoPath, pointer, err) + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", repoPath, key, err) } contentLocation = strings.TrimSpace(contentLocation) contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals @@ -122,11 +50,11 @@ func ContentLocationFromPointer(repoPath, pointer string) (string, error) { // return the absolute path of the content pointed to by the annex pointer stored in the git object // errors if the content is not found in this repo func ContentLocation(blob *git.Blob) (string, error) { - pointer, err := Pointer(blob) + key, err := LookupKey(blob) if err != nil { return "", err } - return ContentLocationFromPointer(blob.Repo().Path, pointer) + return ContentLocationFromKey(blob.Repo().Path, key) } // returns a stream open to the annex content @@ -147,11 +75,11 @@ func IsAnnexed(blob *git.Blob) (bool, error) { return false, nil } - // Pointer() is written to only return well-formed pointers + // LookupKey is written to only return well-formed keys // so the test is just to see if it errors - _, err := Pointer(blob) + _, err := LookupKey(blob) if err != nil { - if errors.Is(err, ErrInvalidPointer) { + if errors.Is(err, ErrBlobIsNotAnnexed) { return false, nil } return false, err diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index d09dc558a2..14792af066 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -2576,16 +2576,16 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { if err != nil { return err } - key, err := annex.Pointer(blob) + key, err := annex.LookupKey(blob) if err != nil { return err } - localObjectPath, err := annex.ContentLocationFromPointer(repoPath, key) + localObjectPath, err := annex.ContentLocationFromKey(repoPath, key) if err != nil { return err } - remoteObjectPath, err := annex.ContentLocationFromPointer(remoteRepoPath, key) + remoteObjectPath, err := annex.ContentLocationFromKey(remoteRepoPath, key) if err != nil { return err } From 5b3d77e3b2c6849980b1bd2bd860edec38fd581c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 29 Oct 2024 11:45:40 +0000 Subject: [PATCH 36/47] Add an OCI image build and publish workflow (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #49. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/50 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- .forgejo/workflows/build-oci-image.yml | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .forgejo/workflows/build-oci-image.yml diff --git a/.forgejo/workflows/build-oci-image.yml b/.forgejo/workflows/build-oci-image.yml new file mode 100644 index 0000000000..9eabdf3eab --- /dev/null +++ b/.forgejo/workflows/build-oci-image.yml @@ -0,0 +1,37 @@ +on: + push: + branches: + - 'forgejo' + tags: + - '*-git-annex*' + +jobs: + build-oci-image: + runs-on: docker + strategy: + matrix: + type: ["rootful", "rootless"] + steps: + - name: Determine registry and username + id: determine-registry-and-username + run: | + echo "registry=${GITHUB_SERVER_URL#https://}" >> "$GITHUB_OUTPUT" + echo "username=${GITHUB_REPOSITORY%/*}" >> "$GITHUB_OUTPUT" + - name: Install Docker + run: curl -fsSL https://get.docker.com | sh + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ${{ steps.determine-registry-and-username.outputs.registry }} + username: ${{ steps.determine-registry-and-username.outputs.username }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: ${{ (matrix.type == 'rootful' && 'Dockerfile') || (matrix.type == 'rootless' && 'Dockerfile.rootless') }} + push: true + tags: ${{ steps.determine-registry-and-username.outputs.registry }}/${{ github.repository }}:${{ github.ref_name }}${{ (matrix.type == 'rootful' && ' ') || (matrix.type == 'rootless' && '-rootless') }} From c659e4befad53393eee2a2d545e91b81278656fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Sat, 2 Nov 2024 15:00:59 +0000 Subject: [PATCH 37/47] Fix Forgejo version in published OCI images (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Forgejo version is derived from the git history, so the image build needs to happen in the context of a full repository clone. Also, the post-processing of the version string needs to remove the second occurrence of "-g", as the first one is now part of the added "-git-annex" part. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/51 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- .forgejo/workflows/build-oci-image.yml | 4 ++++ Makefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build-oci-image.yml b/.forgejo/workflows/build-oci-image.yml index 9eabdf3eab..8e843b41ee 100644 --- a/.forgejo/workflows/build-oci-image.yml +++ b/.forgejo/workflows/build-oci-image.yml @@ -12,6 +12,9 @@ jobs: matrix: type: ["rootful", "rootless"] steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch the full history so that the Forgejo version is determined properly - name: Determine registry and username id: determine-registry-and-username run: | @@ -32,6 +35,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v6 with: + context: . file: ${{ (matrix.type == 'rootful' && 'Dockerfile') || (matrix.type == 'rootless' && 'Dockerfile.rootless') }} push: true tags: ${{ steps.determine-registry-and-username.outputs.registry }}/${{ github.repository }}:${{ github.ref_name }}${{ (matrix.type == 'rootful' && ' ') || (matrix.type == 'rootless' && '-rootless') }} diff --git a/Makefile b/Makefile index 66f8524c04..b1272b640f 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ else FORGEJO_VERSION_API ?= $(GITEA_VERSION)+${GITEA_COMPATIBILITY} else # drop the "g" prefix prepended by git describe to the commit hash - FORGEJO_VERSION ?= $(shell git describe --exclude '*-test' --tags --always | sed 's/^v//' | sed 's/\-g/-/')+${GITEA_COMPATIBILITY} + FORGEJO_VERSION ?= $(shell git describe --exclude '*-test' --tags --always | sed 's/^v//' | sed 's/\-g/-/2')+${GITEA_COMPATIBILITY} endif endif FORGEJO_VERSION_MAJOR=$(shell echo $(FORGEJO_VERSION) | sed -e 's/\..*//') From af3febc6932fefb64e444d80b538516f28cf29c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 6 Nov 2024 14:29:39 +0000 Subject: [PATCH 38/47] Explicitly set http(s) default ports in annex.url (#55) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise, git-annex tries to use its own default port (9417) and fails. Fixes #52. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/55 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- routers/web/repo/githttp.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index 3e80e120b5..58a14fe26c 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -10,6 +10,7 @@ import ( gocontext "context" "fmt" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -557,7 +558,23 @@ func GetConfig(ctx *context.Context) { return } if !setting.Annex.DisableP2PHTTP { - config = append(config, []byte("[annex]\n\turl = annex+"+setting.AppURL+"git-annex-p2phttp\n")...) + appURL, err := url.Parse(setting.AppURL) + if err != nil { + log.Error("Could not parse 'setting.AppURL': %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + if appURL.Port() == "" { + // If there is no port set then set the http(s) default ports. + // Without this, git-annex would try its own default port (9417) and fail. + if appURL.Scheme == "http" { + appURL.Host += ":80" + } + if appURL.Scheme == "https" { + appURL.Host += ":443" + } + } + config = append(config, []byte("[annex]\n\turl = annex+"+appURL.String()+"git-annex-p2phttp\n")...) } ctx.Resp.Header().Set("Content-Type", "text/plain") ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(config))) From 1dd008f8e19e6821d5bc993b28ae3823759f07cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 6 Nov 2024 16:22:29 +0000 Subject: [PATCH 39/47] Cache git-annex UUID to repository path mappings (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Always walking the filesystem and searching for UUIDs slowed p2phttp operations down significantly on a production server with more than a handful of repositories. This caching strategy ensures that only the first call is rather slow, and subsequent ones should be much faster. This should better be implemented as a background job, but for now this is a simple solution to the problem. Fixes #53. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/54 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 5d239f7752..f9e50094f9 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -95,25 +95,43 @@ func IsAnnexRepo(repo *git.Repository) bool { var repoConfigFileRe = regexp.MustCompile("[^/]+/[^/]+.git/config$") -func UUID2RepoPath(uuid string) (string, error) { - var repoPath string - err := filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { +var ( + uuid2repoPathCache = make(map[string]string) + repoPath2uuidCache = make(map[string]string) +) + +func updateUUID2RepoPathCache() error { + return filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { if err == nil && repoConfigFileRe.MatchString(path) { thisRepoPath := strings.TrimSuffix(path, "/config") + _, ok := repoPath2uuidCache[thisRepoPath] + if ok { + return nil + } stdout, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: thisRepoPath}) if err != nil { return nil } repoUUID := strings.TrimSpace(stdout) - if repoUUID == uuid { - repoPath = thisRepoPath - return fs.SkipAll + if repoUUID != "" { + uuid2repoPathCache[repoUUID] = thisRepoPath + repoPath2uuidCache[thisRepoPath] = repoUUID } } return nil }) - if err != nil { +} + +func UUID2RepoPath(uuid string) (string, error) { + if repoPath, ok := uuid2repoPathCache[uuid]; ok { + return repoPath, nil + } + // If the cache didn't contain an entry for the UUID then update the cache and try again + if err := updateUUID2RepoPathCache(); err != nil { return "", err } - return repoPath, nil + if repoPath, ok := uuid2repoPathCache[uuid]; ok { + return repoPath, nil + } + return "", fmt.Errorf("no repository known for UUID '%s'", uuid) } From ebac40612799ec97ac445fe5e588add46207f8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 30 Jan 2025 10:50:48 +0000 Subject: [PATCH 40/47] Pre-populate the git-annex UUID cache at startup (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This eliminates the wait time for the first p2phttp connection since server startup at the cost of adding that time to the startup itself. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/59 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 9 +++++++++ routers/init.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index f9e50094f9..4dabd1d415 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -20,6 +20,7 @@ import ( "strings" "forgejo.org/modules/git" + "forgejo.org/modules/log" "forgejo.org/modules/setting" ) @@ -100,6 +101,14 @@ var ( repoPath2uuidCache = make(map[string]string) ) +func Init() error { + if !setting.Annex.Enabled { + return nil + } + log.Info("Populating the git-annex UUID cache with existing repositories") + return updateUUID2RepoPathCache() +} + func updateUUID2RepoPathCache() error { return filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { if err == nil && repoConfigFileRe.MatchString(path) { diff --git a/routers/init.go b/routers/init.go index 90a1cb1e89..5f9dbbb4dd 100644 --- a/routers/init.go +++ b/routers/init.go @@ -11,6 +11,7 @@ import ( "forgejo.org/models" asymkey_model "forgejo.org/models/asymkey" authmodel "forgejo.org/models/auth" + "forgejo.org/modules/annex" "forgejo.org/modules/cache" "forgejo.org/modules/eventsource" "forgejo.org/modules/git" @@ -167,6 +168,8 @@ func InitWebInstalled(ctx context.Context) { actions_service.Init() + mustInit(annex.Init) + // Finally start up the cron cron.NewContext(ctx) } From 5bcf4c1821ab9e8abd4fcdd5601f34a2b96d21ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 30 Jan 2025 16:44:59 +0000 Subject: [PATCH 41/47] Invalidate outdated annex UUID cache entries (#60) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation could lead to errors e.g. when a repository was removed and a new one with a new UUID was created under the same name. This now checks the validity of the retrieved cache entry every time and invalidates the cache if necessary. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/60 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 4dabd1d415..09c16809f7 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -131,7 +131,7 @@ func updateUUID2RepoPathCache() error { }) } -func UUID2RepoPath(uuid string) (string, error) { +func repoPathFromUUIDCache(uuid string) (string, error) { if repoPath, ok := uuid2repoPathCache[uuid]; ok { return repoPath, nil } @@ -144,3 +144,37 @@ func UUID2RepoPath(uuid string) (string, error) { } return "", fmt.Errorf("no repository known for UUID '%s'", uuid) } + +func checkValidity(uuid, repoPath string) (bool, error) { + stdout, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return false, err + } + repoUUID := strings.TrimSpace(stdout) + return uuid == repoUUID, nil +} + +func removeCachedEntries(uuid, repoPath string) { + delete(uuid2repoPathCache, uuid) + delete(repoPath2uuidCache, repoPath) +} + +func UUID2RepoPath(uuid string) (string, error) { + // Get the current cache entry for the UUID + repoPath, err := repoPathFromUUIDCache(uuid) + if err != nil { + return "", err + } + // Check if it is still up-to-date + valid, err := checkValidity(uuid, repoPath) + if err != nil { + return "", err + } + if !valid { + // If it isn't, remove the cache entry and try again + removeCachedEntries(uuid, repoPath) + return UUID2RepoPath(uuid) + } + // Otherwise just return the cached entry + return repoPath, nil +} From a0ca7bfb6b3136ced6ee78d35ea4e47367ff7deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 31 Jan 2025 00:43:20 +0000 Subject: [PATCH 42/47] Use annexed content for comparison in diffs (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it such that annexed files are treated like plain git files in comparisons (e.g. the diff of a commit). It also changes the image diff viewer to show a more reasonable error message when one of the annexed files under comparison is missing. Fixes #56. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/57 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 12 ++++++++++++ routers/web/repo/compare.go | 27 +++++++++++++++++++++------ web_src/js/features/imagediff.js | 12 +++++++++++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 09c16809f7..7c3e47acb6 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -22,6 +22,7 @@ import ( "forgejo.org/modules/git" "forgejo.org/modules/log" "forgejo.org/modules/setting" + "forgejo.org/modules/typesniffer" ) // ErrBlobIsNotAnnexed occurs if a blob does not contain a valid annex key @@ -178,3 +179,14 @@ func UUID2RepoPath(uuid string) (string, error) { // Otherwise just return the cached entry return repoPath, nil } + +// GuessContentType guesses the content type of the annexed blob. +func GuessContentType(blob *git.Blob) (typesniffer.SniffedType, error) { + r, err := Content(blob) + if err != nil { + return typesniffer.SniffedType{}, err + } + defer r.Close() + + return typesniffer.DetectContentTypeFromReader(r) +} diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index db65e889e0..55d50c9a79 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -23,6 +23,7 @@ import ( repo_model "forgejo.org/models/repo" "forgejo.org/models/unit" user_model "forgejo.org/models/user" + "forgejo.org/modules/annex" "forgejo.org/modules/base" "forgejo.org/modules/charset" csv_module "forgejo.org/modules/csv" @@ -72,7 +73,21 @@ func setCompareContext(ctx *context.Context, before, head *git.Commit, headOwner return st } - st, err := blob.GuessContentType() + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + log.Error("IsAnnexed failed: %v", err) + return st + } + if isAnnexed { + st, err = annex.GuessContentType(blob) + if err != nil { + log.Error("GuessContentType failed: %v", err) + return st + } + return st + } + + st, err = blob.GuessContentType() if err != nil { log.Error("GuessContentType failed: %v", err) return st @@ -90,18 +105,18 @@ func SourceCommitURL(owner, name string, commit *git.Commit) string { return setting.AppSubURL + "/" + url.PathEscape(owner) + "/" + url.PathEscape(name) + "/src/commit/" + url.PathEscape(commit.ID.String()) } -// RawCommitURL creates a relative URL for the raw commit in the given repository -func RawCommitURL(owner, name string, commit *git.Commit) string { - return setting.AppSubURL + "/" + url.PathEscape(owner) + "/" + url.PathEscape(name) + "/raw/commit/" + url.PathEscape(commit.ID.String()) +// MediaCommitURL creates a relative URL for the commit media (plain git, LFS, or annex content) in the given repository +func MediaCommitURL(owner, name string, commit *git.Commit) string { + return setting.AppSubURL + "/" + url.PathEscape(owner) + "/" + url.PathEscape(name) + "/media/commit/" + url.PathEscape(commit.ID.String()) } // setPathsCompareContext sets context data for source and raw paths func setPathsCompareContext(ctx *context.Context, base, head *git.Commit, headOwner, headName string) { ctx.Data["SourcePath"] = SourceCommitURL(headOwner, headName, head) - ctx.Data["RawPath"] = RawCommitURL(headOwner, headName, head) + ctx.Data["RawPath"] = MediaCommitURL(headOwner, headName, head) if base != nil { ctx.Data["BeforeSourcePath"] = SourceCommitURL(headOwner, headName, base) - ctx.Data["BeforeRawPath"] = RawCommitURL(headOwner, headName, base) + ctx.Data["BeforeRawPath"] = MediaCommitURL(headOwner, headName, base) } } diff --git a/web_src/js/features/imagediff.js b/web_src/js/features/imagediff.js index d1b139ffde..52c9017b07 100644 --- a/web_src/js/features/imagediff.js +++ b/web_src/js/features/imagediff.js @@ -92,7 +92,17 @@ export function initImageDiff() { return loadElem(img, info.path); })); // only the first images is associated with $boundsInfo - if (!success) info.$boundsInfo.text('(image error)'); + if (!success) { + const blobContent = await GET(info.path.replace('/media/', '/raw/')).then((response) => response.text()); + if (blobContent.startsWith('.git/annex/objects')) { + for (const item of document.querySelectorAll('.image-diff .overflow-menu-items .item')) { + item.style.display = 'none'; + } + info.$boundsInfo[0].parentElement.textContent = 'annexed file is not present on the server'; + } else { + info.$boundsInfo.text('(image error)'); + } + } if (info.mime === 'image/svg+xml') { const resp = await GET(info.path); const text = await resp.text(); From aba710a96293cbfde8c908584c8162408733dfbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 21 Feb 2025 13:59:06 +0000 Subject: [PATCH 43/47] fix: improve git-annex UUID cache update times (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Globbing for the config files is marginally faster than walking the directory and checking for config files. Replacing the expensive calls to `git config` for each repository with reading the repository's config as an ini file is two orders of magnitude faster. All in all this reduces the required time initializing the cache for approx. 3000 repositories from approx. 5s to 50ms. The server startup now also logs how long the cache update took and the cache update is only done if p2phttp support is not disabled, because p2phttp support is currently the only feature that requires the UUID cache. Fixes #63, fixes #64. The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. - [x] I do not want this change to show in the release notes. - [ ] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/65 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 56 ++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 7c3e47acb6..6fc2942e82 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -12,17 +12,18 @@ package annex import ( "errors" "fmt" - "io/fs" "os" "path" "path/filepath" - "regexp" "strings" + "time" "forgejo.org/modules/git" "forgejo.org/modules/log" "forgejo.org/modules/setting" "forgejo.org/modules/typesniffer" + + "gopkg.in/ini.v1" //nolint:depguard // This import is forbidden in favor of using the setting module, but we need ini parsing for something other than Forgejo settings ) // ErrBlobIsNotAnnexed occurs if a blob does not contain a valid annex key @@ -95,8 +96,6 @@ func IsAnnexRepo(repo *git.Repository) bool { return err == nil } -var repoConfigFileRe = regexp.MustCompile("[^/]+/[^/]+.git/config$") - var ( uuid2repoPathCache = make(map[string]string) repoPath2uuidCache = make(map[string]string) @@ -106,30 +105,39 @@ func Init() error { if !setting.Annex.Enabled { return nil } - log.Info("Populating the git-annex UUID cache with existing repositories") - return updateUUID2RepoPathCache() + if !setting.Annex.DisableP2PHTTP { + log.Info("Populating the git-annex UUID cache with existing repositories") + start := time.Now() + if err := updateUUID2RepoPathCache(); err != nil { + return err + } + log.Info("Populating the git-annex UUID cache took %v", time.Since(start)) + } + return nil } func updateUUID2RepoPathCache() error { - return filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { - if err == nil && repoConfigFileRe.MatchString(path) { - thisRepoPath := strings.TrimSuffix(path, "/config") - _, ok := repoPath2uuidCache[thisRepoPath] - if ok { - return nil - } - stdout, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: thisRepoPath}) - if err != nil { - return nil - } - repoUUID := strings.TrimSpace(stdout) - if repoUUID != "" { - uuid2repoPathCache[repoUUID] = thisRepoPath - repoPath2uuidCache[thisRepoPath] = repoUUID - } + configFiles, err := filepath.Glob(filepath.Join(setting.RepoRootPath, "*", "*", "config")) + if err != nil { + return err + } + for _, configFile := range configFiles { + repoPath := strings.TrimSuffix(configFile, "/config") + _, ok := repoPath2uuidCache[repoPath] + if ok { + continue } - return nil - }) + config, err := ini.Load(configFile) + if err != nil { + continue + } + repoUUID := config.Section("annex").Key("uuid").Value() + if repoUUID != "" { + uuid2repoPathCache[repoUUID] = repoPath + repoPath2uuidCache[repoPath] = repoUUID + } + } + return nil } func repoPathFromUUIDCache(uuid string) (string, error) { From a4dd1bcc4bbd37c41d10bca432b2248e8c4120a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 21 Feb 2025 21:31:48 +0000 Subject: [PATCH 44/47] feat: copy annexed files on pull request merge (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes the PR merge process such that annexed files contained in the to-be-merged commits are copied from the head repository to the base repository as part of the merge, similar to how it is done for LFS files. Fixes #11. The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [ ] I did not document these changes and I do not expect someone else to do it. - [x] I do not want this change to show in the release notes. - [ ] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/62 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 56 +++++++++++++ modules/git/pipeline/catfile.go | 33 ++++++++ services/pull/annex.go | 61 ++++++++++++++ services/pull/merge.go | 7 ++ tests/integration/git_annex_test.go | 126 ++++++++++++++++++++-------- 5 files changed, 249 insertions(+), 34 deletions(-) create mode 100644 services/pull/annex.go diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 6fc2942e82..26775581ca 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -10,12 +10,16 @@ package annex import ( + "bytes" + "context" "errors" "fmt" + "io" "os" "path" "path/filepath" "strings" + "sync" "time" "forgejo.org/modules/git" @@ -29,6 +33,16 @@ import ( // ErrBlobIsNotAnnexed occurs if a blob does not contain a valid annex key var ErrBlobIsNotAnnexed = errors.New("not a git-annex pointer") +func PrivateInit(ctx context.Context, repoPath string) error { + if _, _, err := git.NewCommand(ctx, "config", "annex.private", "true").RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + return err + } + if _, _, err := git.NewCommand(ctx, "annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + return err + } + return nil +} + func LookupKey(blob *git.Blob) (string, error) { stdout, _, err := git.NewCommand(git.DefaultContext, "annex", "lookupkey", "--ref").AddDynamicArguments(blob.ID.String()).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) if err != nil { @@ -38,6 +52,42 @@ func LookupKey(blob *git.Blob) (string, error) { return key, nil } +// LookupKeyBatch runs git annex lookupkey --batch --ref +func LookupKeyBatch(ctx context.Context, shasToBatchReader *io.PipeReader, lookupKeyBatchWriter *io.PipeWriter, wg *sync.WaitGroup, repoPath string) { + defer wg.Done() + defer shasToBatchReader.Close() + defer lookupKeyBatchWriter.Close() + + stderr := new(bytes.Buffer) + var errbuf strings.Builder + if err := git.NewCommand(ctx, "annex", "lookupkey", "--batch", "--ref").Run(&git.RunOpts{ + Dir: repoPath, + Stdout: lookupKeyBatchWriter, + Stdin: shasToBatchReader, + Stderr: stderr, + }); err != nil { + _ = lookupKeyBatchWriter.CloseWithError(fmt.Errorf("git annex lookupkey --batch --ref [%s]: %w - %s", repoPath, err, errbuf.String())) + } +} + +// CopyFromToBatch runs git -c annex.hardlink=true annex copy --batch-keys --from --to +func CopyFromToBatch(ctx context.Context, from, to string, keysToCopyReader *io.PipeReader, wg *sync.WaitGroup, repoPath string) { + defer wg.Done() + defer keysToCopyReader.Close() + + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + var errbuf strings.Builder + if err := git.NewCommand(ctx, "-c", "annex.hardlink=true", "annex", "copy", "--batch-keys", "--from").AddDynamicArguments(from).AddArguments("--to").AddDynamicArguments(to).Run(&git.RunOpts{ + Dir: repoPath, + Stdout: stdout, + Stdin: keysToCopyReader, + Stderr: stderr, + }); err != nil { + _ = keysToCopyReader.CloseWithError(fmt.Errorf("git annex copy --batch-keys --from --to [%s]: %w - %s", repoPath, err, errbuf.String())) + } +} + func ContentLocationFromKey(repoPath, key string) (string, error) { contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(key).RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { @@ -90,6 +140,12 @@ func IsAnnexed(blob *git.Blob) (bool, error) { return true, nil } +// PathIsAnnexRepo determines if repoPath is a git-annex enabled repository +func PathIsAnnexRepo(repoPath string) bool { + _, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + return err == nil +} + // IsAnnexRepo determines if repo is a git-annex enabled repository func IsAnnexRepo(repo *git.Repository) bool { _, _, err := git.NewCommand(repo.Ctx, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repo.Path}) diff --git a/modules/git/pipeline/catfile.go b/modules/git/pipeline/catfile.go index 476f876e2b..6ada51ae82 100644 --- a/modules/git/pipeline/catfile.go +++ b/modules/git/pipeline/catfile.go @@ -106,3 +106,36 @@ func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, s } } } + +// BlobsLessThanOrEqual32KiBFromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <=32KiB in size +func BlobsLessThanOrEqual32KiBFromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { + defer wg.Done() + defer catFileCheckReader.Close() + scanner := bufio.NewScanner(catFileCheckReader) + defer func() { + _ = shasToBatchWriter.CloseWithError(scanner.Err()) + }() + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 3 || fields[1] != "blob" { + continue + } + size, _ := strconv.Atoi(fields[2]) + if size > 32*1024 { + continue + } + toWrite := []byte(fields[0] + "\n") + for len(toWrite) > 0 { + n, err := shasToBatchWriter.Write(toWrite) + if err != nil { + _ = catFileCheckReader.CloseWithError(err) + break + } + toWrite = toWrite[n:] + } + } +} diff --git a/services/pull/annex.go b/services/pull/annex.go new file mode 100644 index 0000000000..67985aa1b7 --- /dev/null +++ b/services/pull/annex.go @@ -0,0 +1,61 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pull + +import ( + "context" + "io" + "sync" + + "forgejo.org/modules/annex" + "forgejo.org/modules/git/pipeline" +) + +// AnnexPush copies all annexed files referenced in new commits from the head repository to the base repository +func AnnexPush(ctx context.Context, tmpBasePath, mergeHeadSHA, mergeBaseSHA string) error { + // Initialize the temporary repository with git-annex + if err := annex.PrivateInit(ctx, tmpBasePath); err != nil { + return err + } + + revListReader, revListWriter := io.Pipe() + shasToCheckReader, shasToCheckWriter := io.Pipe() + catFileCheckReader, catFileCheckWriter := io.Pipe() + shasToBatchReader, shasToBatchWriter := io.Pipe() + lookupKeyBatchReader, lookupKeyBatchWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(6) + // Create the go-routines in reverse order. + + // 6. Take the referenced keys and copy their data from the head repository to + // the base repository + go annex.CopyFromToBatch(ctx, "head_repo", "origin", lookupKeyBatchReader, &wg, tmpBasePath) + + // 5. Take the shas of the blobs and resolve them to annex keys, git-annex + // should filter out anything that doesn't reference a key + go annex.LookupKeyBatch(ctx, shasToBatchReader, lookupKeyBatchWriter, &wg, tmpBasePath) + + // 4. From the provided objects restrict to blobs <=32KiB + go pipeline.BlobsLessThanOrEqual32KiBFromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) + + // 3. Run batch-check on the objects retrieved from rev-list + go pipeline.CatFileBatchCheck(ctx, shasToCheckReader, catFileCheckWriter, &wg, tmpBasePath) + + // 2. Check each object retrieved rejecting those without names as they will be commits or trees + go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) + + // 1. Run rev-list objects from mergeHead to mergeBase + go pipeline.RevListObjects(ctx, revListWriter, &wg, tmpBasePath, mergeHeadSHA, mergeBaseSHA, errChan) + + wg.Wait() + select { + case err, has := <-errChan: + if has { + return err + } + default: + } + return nil +} diff --git a/services/pull/merge.go b/services/pull/merge.go index 9b0d632377..2f6ee6754c 100644 --- a/services/pull/merge.go +++ b/services/pull/merge.go @@ -22,6 +22,7 @@ import ( repo_model "forgejo.org/models/repo" "forgejo.org/models/unit" user_model "forgejo.org/models/user" + "forgejo.org/modules/annex" "forgejo.org/modules/cache" "forgejo.org/modules/git" "forgejo.org/modules/log" @@ -314,6 +315,12 @@ func doMergeAndPush(ctx context.Context, pr *issues_model.PullRequest, doer *use } } + if setting.Annex.Enabled && annex.PathIsAnnexRepo(pr.BaseRepo.RepoPath()) && annex.PathIsAnnexRepo(pr.HeadRepo.RepoPath()) { + if err := AnnexPush(ctx, mergeCtx.tmpBasePath, mergeHeadSHA, mergeBaseSHA); err != nil { + return "", err + } + } + var headUser *user_model.User err = pr.HeadRepo.LoadOwner(ctx) if err != nil { diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 14792af066..1b832db1fb 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -28,9 +28,11 @@ import ( "forgejo.org/modules/git" "forgejo.org/modules/setting" api "forgejo.org/modules/structs" + "forgejo.org/modules/test" "forgejo.org/modules/util" "forgejo.org/tests" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -62,6 +64,95 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, return nil } +func TestGitAnnexPullRequest(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + defer tests.PrepareTestEnv(t)() + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + upstreamRepoName := "annex-pull-request-test-" + objectFormat.Name() + forkRepoName := upstreamRepoName + ctx := NewAPITestContext(t, "user2", upstreamRepoName, auth_model.AccessTokenScopeWriteRepository) + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) + session := loginUser(t, "user1") + testRepoFork(t, session, "user2", upstreamRepoName, "user1", forkRepoName) + + // Generate random file + tmpFile := path.Join(t.TempDir(), "somefile") + require.NoError(t, generateRandomFile(1024*1024/4, tmpFile)) + expectedContent, err := os.ReadFile(tmpFile) + require.NoError(t, err) + + testUploadFile(t, session, "user1", forkRepoName, setting.Repository.DefaultBranch, filepath.Base(tmpFile), tmpFile) + + resp := testPullCreate(t, session, "user1", forkRepoName, false, setting.Repository.DefaultBranch, setting.Repository.DefaultBranch, "Testing git-annex content in a pull request") + + elem := strings.Split(test.RedirectURL(resp), "/") + assert.EqualValues(t, "pulls", elem[3]) + testPullMerge(t, session, elem[1], elem[2], elem[4], repo_model.MergeStyleMerge, false) + + // Get some handles on the target repository and file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + repo, err := git.OpenRepository(git.DefaultContext, remoteRepoPath) + require.NoError(t, err) + defer repo.Close() + tree, err := repo.GetTree(setting.Repository.DefaultBranch) + require.NoError(t, err) + treeEntry, err := tree.GetTreeEntryByPath(filepath.Base(tmpFile)) + require.NoError(t, err) + blob := treeEntry.Blob() + + // Check that the pull request file is annexed + isAnnexed, err := annex.IsAnnexed(blob) + require.NoError(t, err) + require.True(t, isAnnexed) + + // Check that the pull request file has the correct content + annexedFile, err := annex.Content(blob) + require.NoError(t, err) + actualContent, err := io.ReadAll(annexedFile) + require.NoError(t, err) + require.Equal(t, expectedContent, actualContent) + }) + }) +} + +func testUploadFile(t *testing.T, session *TestSession, username, reponame, branch, filename, path string) { + t.Helper() + + body := &bytes.Buffer{} + mpForm := multipart.NewWriter(body) + err := mpForm.WriteField("_csrf", GetCSRF(t, session, username+"/"+reponame+"/_upload/"+branch)) + require.NoError(t, err) + + file, err := mpForm.CreateFormFile("file", filename) + require.NoError(t, err) + + srcFile, err := os.Open(path) + require.NoError(t, err) + + io.Copy(file, srcFile) + require.NoError(t, mpForm.Close()) + + req := NewRequestWithBody(t, "POST", "/"+username+"/"+reponame+"/upload-file", body) + req.Header.Add("Content-Type", mpForm.FormDataContentType()) + resp := session.MakeRequest(t, req, http.StatusOK) + + respMap := map[string]string{} + DecodeJSON(t, resp, &respMap) + fileUUID := respMap["uuid"] + + req = NewRequestWithValues(t, "POST", username+"/"+reponame+"/_upload/"+branch, map[string]string{ + "commit_choice": "direct", + "files": fileUUID, + "_csrf": GetCSRF(t, session, username+"/"+reponame+"/_upload/"+branch), + "commit_mail_id": "-1", + }) + session.MakeRequest(t, req, http.StatusSeeOther) +} + func TestGitAnnexWebUpload(t *testing.T) { if !setting.Annex.Enabled { t.Skip("Skipping since annex support is disabled.") @@ -72,32 +163,6 @@ func TestGitAnnexWebUpload(t *testing.T) { ctx := NewAPITestContext(t, "user2", "annex-web-upload-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) - uploadFile := func(t *testing.T, path string) string { - t.Helper() - - body := &bytes.Buffer{} - mpForm := multipart.NewWriter(body) - err := mpForm.WriteField("_csrf", GetCSRF(t, ctx.Session, ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch)) - require.NoError(t, err) - - file, err := mpForm.CreateFormFile("file", filepath.Base(path)) - require.NoError(t, err) - - srcFile, err := os.Open(path) - require.NoError(t, err) - - io.Copy(file, srcFile) - require.NoError(t, mpForm.Close()) - - req := NewRequestWithBody(t, "POST", "/"+ctx.Username+"/"+ctx.Reponame+"/upload-file", body) - req.Header.Add("Content-Type", mpForm.FormDataContentType()) - resp := ctx.Session.MakeRequest(t, req, http.StatusOK) - - respMap := map[string]string{} - DecodeJSON(t, resp, &respMap) - return respMap["uuid"] - } - // Generate random file tmpFile := path.Join(t.TempDir(), "web-upload-test-file.bin") require.NoError(t, generateRandomFile(1024*1024/4, tmpFile)) @@ -105,14 +170,7 @@ func TestGitAnnexWebUpload(t *testing.T) { require.NoError(t, err) // Upload generated file - fileUUID := uploadFile(t, tmpFile) - req := NewRequestWithValues(t, "POST", ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch, map[string]string{ - "commit_choice": "direct", - "files": fileUUID, - "_csrf": GetCSRF(t, ctx.Session, ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch), - "commit_mail_id": "-1", - }) - ctx.Session.MakeRequest(t, req, http.StatusSeeOther) + testUploadFile(t, ctx.Session, ctx.Username, ctx.Reponame, setting.Repository.DefaultBranch, filepath.Base(tmpFile), tmpFile) // Get some handles on the target repository and file remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) From 8e1967484000f81b32f6de9c6e0f3a413b1056bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 17 Mar 2025 09:58:47 +0000 Subject: [PATCH 45/47] fix: set git identity for p2phttp processes (#70) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In some situations it could happen that `git annex p2phttp` needs some kind of maintenance work resulting in a commit, but without a configured git identity p2phttp would refuse to run. This could break p2phttp support. Setting `GIT_AUTHOR_{NAME,EMAIL}` should remedy this issue. Fixes #69. The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. - [x] I do not want this change to show in the release notes. - [ ] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/70 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- routers/web/repo/annex.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/routers/web/repo/annex.go b/routers/web/repo/annex.go index facdded335..191ad5dd20 100644 --- a/routers/web/repo/annex.go +++ b/routers/web/repo/annex.go @@ -19,6 +19,7 @@ import ( "forgejo.org/modules/annex" "forgejo.org/modules/graceful" "forgejo.org/modules/log" + "forgejo.org/modules/setting" services_context "forgejo.org/services/context" ) @@ -93,6 +94,10 @@ func AnnexP2PHTTP(ctx *services_context.Context) { Pdeathsig: syscall.SIGINT, } cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME="+setting.AppName, + "GIT_AUTHOR_EMAIL="+setting.RunUser+"@"+setting.Domain, + ) _ = cmd.Run() }(p2phttpCtx) graceful.GetManager().RunAtTerminate(p2phttpCtxCancel) From 19be11668b34d994c6dd59d7ebda47ec7995374c Mon Sep 17 00:00:00 2001 From: matrss Date: Thu, 10 Apr 2025 12:52:29 +0000 Subject: [PATCH 46/47] Forgejo-aneksajo branding (#71) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original logo was created by Caesar Schinas and is licensed under the Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0) license. This implies permission to remix. Source: https://codeberg.org/forgejo/governance/src/branch/main/branding#logo The git-annex logo is covered by ``` Copyright: 2007 Henrik Nyh 2010 Joey Hess 2013 John Lawrence 2024 Yann Büchau License: other Free to modify and redistribute with due credit, and obviously free to use. ``` Source: https://git-annex.branchable.com/logo/ Co-authored-by: Michael Hanke Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/71 --- assets/favicon.svg | 56 +++++++++++++++++++++++++--------------------- assets/logo.svg | 56 +++++++++++++++++++++++++--------------------- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/assets/favicon.svg b/assets/favicon.svg index bcacdc0200..bb0031b93d 100644 --- a/assets/favicon.svg +++ b/assets/favicon.svg @@ -1,27 +1,33 @@ - - - - - - - - + + + + + + + + + + + + + + + + + + diff --git a/assets/logo.svg b/assets/logo.svg index bcacdc0200..bb0031b93d 100644 --- a/assets/logo.svg +++ b/assets/logo.svg @@ -1,27 +1,33 @@ - - - - - - - - + + + + + + + + + + + + + + + + + + From 7bbc84ed5eeca5cd548605647f5cd9cc1d9258f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Sat, 24 May 2025 20:39:34 +0200 Subject: [PATCH 47/47] fix: set committer identity for p2phttp processes (#76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just setting GIT_AUTHOR_{NAME,EMAIL} wasn't enough, GIT_COMMITTER_{NAME,EMAIL} is also required. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/76 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- routers/web/repo/annex.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/web/repo/annex.go b/routers/web/repo/annex.go index 191ad5dd20..fa4d1c6ba4 100644 --- a/routers/web/repo/annex.go +++ b/routers/web/repo/annex.go @@ -97,6 +97,8 @@ func AnnexP2PHTTP(ctx *services_context.Context) { cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME="+setting.AppName, "GIT_AUTHOR_EMAIL="+setting.RunUser+"@"+setting.Domain, + "GIT_COMMITTER_NAME="+setting.AppName, + "GIT_COMMITTER_EMAIL="+setting.RunUser+"@"+setting.Domain, ) _ = cmd.Run() }(p2phttpCtx)