diff --git a/tests/integration/git_annex_archive_test.go b/tests/integration/git_annex_archive_test.go new file mode 100644 index 0000000000..cf1c8f27ac --- /dev/null +++ b/tests/integration/git_annex_archive_test.go @@ -0,0 +1,294 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "path" + "sort" + "strings" + "sync" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/annex" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/require" +) + +type record struct { + name string + isAnnexed bool + annexedMode fs.FileMode + gitMode git.EntryMode + content []byte +} + +func TestGitAnnexArchive(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-archive-test-"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) + req := NewRequest(t, "GET", u.String()) + _ = ctx.Session.MakeRequest(t, req, http.StatusOK) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // get the commitID of the master branch + repo, err := git.OpenRepository(git.DefaultContext, remoteRepoPath) + require.NoError(t, err) + commitID, err := repo.GetBranchCommitID("master") + require.NoError(t, err) + tree, err := repo.GetTree("master") + require.NoError(t, err) + entries, err := tree.ListEntriesRecursiveFast() + require.NoError(t, err) + filesInGit := make(map[string]record, len(entries)) + var annexedMode fs.FileMode + for _, entry := range entries { + if !entry.IsDir() { + name := entry.Name() + blob := entry.Blob() + isAnnexed, err := annex.IsAnnexed(blob) + require.NoError(t, err) + var r io.Reader + if isAnnexed { + fa, err := annex.Content(blob) + require.NoError(t, err) + defer fa.Close() + r = fa + stat, err := fa.Stat() + require.NoError(t, err) + annexedMode = stat.Mode() + } else { + // standard git file + br, err := blob.DataAsync() + require.NoError(t, err) + defer br.Close() + r = br + } + expectedContent, err := io.ReadAll(r) + require.NoError(t, err) + filesInGit[name] = record{ + name: name, + isAnnexed: isAnnexed, + annexedMode: annexedMode, + gitMode: entry.Mode(), + content: expectedContent, + } + } + } + + t.Run("api-v1", func(t *testing.T) { + urlBase := fmt.Sprintf("/api/v1/repos/%s/%s/archive/master", ctx.Username, ctx.Reponame) + doTestArchive(t, ctx, urlBase, commitID, filesInGit) + }) + + t.Run("web", func(t *testing.T) { + urlBase := fmt.Sprintf("/%s/%s/archive/master", ctx.Username, ctx.Reponame) + doTestArchive(t, ctx, urlBase, commitID, filesInGit) + }) + }) + }) +} + +func doTestArchive(t *testing.T, ctx APITestContext, urlBase, commitID string, filesInGit map[string]record) { + // cleanup previously generated archives + adminSession := loginUser(t, "user1") + adminToken := getTokenForLoggedInUser(t, adminSession, auth_model.AccessTokenScopeWriteAdmin) + link, _ := url.Parse("/api/v1/admin/cron/delete_repo_archives") + link.RawQuery = url.Values{"token": {adminToken}}.Encode() + resp := adminSession.MakeRequest(t, NewRequest(t, "POST", link.String()), http.StatusNoContent) + bs, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Empty(t, bs) + + t.Run("TARGZ", func(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + doTestTarGzArchive(t, urlBase+".tar.gz", ctx, commitID, filesInGit) + }() + } + wg.Wait() + }) + t.Run("ZIP", func(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + doTestZipArchive(t, urlBase+".zip", ctx, commitID, filesInGit) + }() + } + wg.Wait() + }) +} + +func getArchiveFromEndpoint(t *testing.T, ctx APITestContext, endpointURL string) []byte { + link, _ := url.Parse(endpointURL) + resp := ctx.Session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK) + bs, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return bs +} + +func doTestTarGzArchive(t *testing.T, endpointURL string, ctx APITestContext, commitID string, filesInGit map[string]record) { + // request a tar.gz archive of the repo + bs := getArchiveFromEndpoint(t, ctx, endpointURL) + + // open the archive for reading + gzrd, err := gzip.NewReader(bytes.NewReader(bs)) + require.NoError(t, err) + defer gzrd.Close() + rd := tar.NewReader(gzrd) + + var filesInArchive []string + for { + header, err := rd.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + + // skip directories + if header.Typeflag == tar.TypeDir { + continue + } + + // check that the pax_global_header comment field is correctly set + if path.Base(header.Name) == "pax_global_header" { + require.Equal(t, commitID, header.PAXRecords["comment"]) + continue // skip the remaining checks since this file does not exist in git + } + + name := strings.TrimPrefix(header.Name, ctx.Reponame+"/") + filesInArchive = append(filesInArchive, name) + + // make sure all files are the same as in the repo itself + actualContent, err := io.ReadAll(rd) + require.NoError(t, err) + actualFileMode := header.FileInfo().Mode() + compareToGitRecord(t, git.TARGZ, filesInGit[name], actualContent, actualFileMode) + } + // check that all files that are in git are also present in the archive + compareListOfStrings(t, mapKeys(filesInGit), filesInArchive) +} + +func doTestZipArchive(t *testing.T, endpointURL string, ctx APITestContext, commitID string, filesInGit map[string]record) { + // request a zip archive of the repo + bs := getArchiveFromEndpoint(t, ctx, endpointURL) + + // open the archive for reading + r, err := zip.NewReader(bytes.NewReader(bs), int64(len(bs))) + require.NoError(t, err) + + // check that the comment field is correctly set + require.Equal(t, commitID, r.Comment) + + var filesInArchive []string + for _, f := range r.File { + // skip directories + if f.FileInfo().IsDir() { + continue + } + + name := strings.TrimPrefix(f.Name, ctx.Reponame+"/") + filesInArchive = append(filesInArchive, name) + + // make sure all files are the same as in the repo itself + frd, err := f.Open() + require.NoError(t, err) + defer frd.Close() + actualContent, err := io.ReadAll(frd) + require.NoError(t, err) + actualFileMode := f.Mode() + compareToGitRecord(t, git.ZIP, filesInGit[name], actualContent, actualFileMode) + } + // check that all files that are in git are also present in the archive + compareListOfStrings(t, mapKeys(filesInGit), filesInArchive) +} + +func compareToGitRecord(t *testing.T, archiveType git.ArchiveType, gitRecord record, actualContent []byte, actualFileMode fs.FileMode) { + expectedContent := gitRecord.content + var expectedFileMode fs.FileMode + // The expected file mode depends on the archive type and the type of file in git + // (regular, executable, symlink, annex pointer, annex symlink) + switch archiveType { + case git.TARGZ: + if gitRecord.gitMode == git.EntryModeExec || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) != 0) { + // If the file is a regular executable git file (plain-git or annex pointer file) + // or a symlink to an executable annexed file expect file mode 0775, just like + // what `git archive` would do for executable files. + expectedFileMode = fs.FileMode(0o775) + } else if gitRecord.gitMode == git.EntryModeBlob || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) == 0) { + // If the file is a regular non-executable git file (plain-git or annex pointer + // file) or a symlink to a non-executable annexed file expect file mode 0664, like + // what `git archive` would do. + expectedFileMode = fs.FileMode(0o664) + } else if gitRecord.gitMode == git.EntryModeSymlink { + // If the file is a plain-git symlink expect a normal symlink. + expectedFileMode = fs.FileMode(0o777) | fs.ModeSymlink + // gitRecord.content contains the link target, but in tar.gz there is no content + // for symlinks, so set the expectedContent to an empty byte slice. + expectedContent = []byte{} + } + case git.ZIP: + if gitRecord.gitMode == git.EntryModeExec || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) != 0) { + // If the file is a regular executable git file (plain-git or annex pointer file) + // or a symlink to an executable annexed file expect file mode 0755. + expectedFileMode = fs.FileMode(0o755) + } else if gitRecord.gitMode == git.EntryModeBlob || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) == 0) { + // If the file is a regular non-executable git file (plain-git or annex pointer + // file) or a symlink to a non-executable annexed file it should be archived with + // creatorFAT and have rw permissions, like what `git archive` would do. + // This means go should read it in with 0666 permissions. + expectedFileMode = fs.FileMode(0o666) + } else if gitRecord.gitMode == git.EntryModeSymlink { + // If the file is a plain-git symlink expect a normal symlink. + expectedFileMode = fs.FileMode(0o777) | fs.ModeSymlink + } + } + + // check that the file modes (type and permissions) are equal + require.Equal(t, expectedFileMode.String(), actualFileMode.String()) + + // check that the contents are equal + require.Equal(t, expectedContent, actualContent) +} + +func compareListOfStrings(t *testing.T, l1, l2 []string) { + sort.Strings(l1) + sort.Strings(l2) + require.Equal(t, l1, l2) +} + +func mapKeys[K comparable, V any](m map[K]V) []K { + keys := make([]K, len(m)) + + i := 0 + for k := range m { + keys[i] = k + i++ + } + + return keys +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index efcb571aa2..b1b0aa72f8 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -2739,6 +2739,16 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a file in a subdirectory + err = os.Mkdir(path.Join(repoPath, "subdir"), 0o755) + if err != nil { + return err + } + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "subdir/subdir-annexed.tiff")) + if err != nil { + return err + } + // // a text file err = os.WriteFile(path.Join(repoPath, "annexed.md"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) if err != nil { @@ -2780,6 +2790,21 @@ func doInitAnnexRepository(repoPath string) error { return err } + // add standard symlinks to git + err = os.Symlink(".gitattributes", path.Join(repoPath, "link-to-.gitattributes")) + if err != nil { + return err + } + err = os.Symlink("missing-file", path.Join(repoPath, "link-to-missing-file")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + // save everything err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex files"}) if err != nil {