forgejo_fusion/services/repository/files/upload.go

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package files

import (
	"context"
	"fmt"
	"html"
	"io"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	git_model "code.gitea.io/gitea/models/git"
	repo_model "code.gitea.io/gitea/models/repo"
	user_model "code.gitea.io/gitea/models/user"
	"code.gitea.io/gitea/modules/annex"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/lfs"
	"code.gitea.io/gitea/modules/setting"
)

// UploadRepoFileOptions contains the uploaded repository file options
type UploadRepoFileOptions struct {
	LastCommitID string
	OldBranch    string
	NewBranch    string
	TreePath     string
	Message      string
	Author       *IdentityOptions
	Committer    *IdentityOptions
	Files        []string // In UUID format
	FullPaths    []string
	Signoff      bool
}

type uploadInfo struct {
	upload        *repo_model.Upload
	lfsMetaObject *git_model.LFSMetaObject
}

func cleanUpAfterFailure(ctx context.Context, infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error {
	for _, info := range *infos {
		if info.lfsMetaObject == nil {
			continue
		}
		if !info.lfsMetaObject.Existing {
			if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, t.repo.ID, info.lfsMetaObject.Oid); err != nil {
				original = fmt.Errorf("%w, %v", original, err) // We wrap the original error - as this is the underlying error that required the fallback
			}
		}
	}
	return original
}

// UploadRepoFiles uploads files to the given repository
func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error {
	if len(opts.Files) == 0 {
		return nil
	}

	if len(opts.Files) != len(opts.FullPaths) {
		return nil
	}

	uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files)
	if err != nil {
		return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err)
	}

	names := make([]string, len(uploads))
	infos := make([]uploadInfo, len(uploads))
	for i, upload := range uploads {
		// Check file is not lfs locked, will return nil if lock setting not enabled
		upload.Name = fileNameSanitize(html.UnescapeString(opts.FullPaths[i]))
		filepath := path.Join(opts.TreePath, upload.Name)
		lfsLock, err := git_model.GetTreePathLock(ctx, repo.ID, filepath)
		if err != nil {
			return err
		}
		if lfsLock != nil && lfsLock.OwnerID != doer.ID {
			u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID)
			if err != nil {
				return err
			}
			return git_model.ErrLFSFileLocked{RepoID: repo.ID, Path: filepath, UserName: u.Name}
		}

		names[i] = upload.Name
		infos[i] = uploadInfo{upload: upload}
	}

	t, err := NewTemporaryUploadRepository(ctx, repo)
	if err != nil {
		return err
	}
	defer t.Close()

	hasOldBranch := true
	if err = t.Clone(opts.OldBranch, false); err != nil {
		if !git.IsErrBranchNotExist(err) || !repo.IsEmpty {
			return err
		}
		if err = t.Init(repo.ObjectFormatName); err != nil {
			return err
		}
		hasOldBranch = false
		opts.LastCommitID = ""
	}
	if hasOldBranch {
		if err = t.SetDefaultIndex(); err != nil {
			return err
		}
	}

	r, err := git.OpenRepository(ctx, repo.RepoPath())
	if err != nil {
		return err
	}
	if annex.IsAnnexRepo(r) {
		// Initialize annex privately in temporary clone
		if err := t.InitPrivateAnnex(); err != nil {
			return err
		}
		// Copy uploaded files into git-annex repository
		if err := copyUploadedFilesIntoAnnexRepository(infos, t, opts.TreePath); err != nil {
			return err
		}
		// Move all annexed content in the temporary repository, i.e. everything we have just added, to the origin
		author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer)
		if err := moveAnnexedFilesToOrigin(t, author, committer); err != nil {
			return err
		}
	} else {
		// Copy uploaded files into repository.
		if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil {
			return err
		}
	}

	// Now write the tree
	treeHash, err := t.WriteTree()
	if err != nil {
		return err
	}

	author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer)

	// Now commit the tree
	commitHash, err := t.CommitTree(opts.LastCommitID, author, committer, treeHash, opts.Message, opts.Signoff)
	if err != nil {
		return err
	}

	// Now deal with LFS objects
	for i := range infos {
		if infos[i].lfsMetaObject == nil {
			continue
		}
		infos[i].lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, infos[i].lfsMetaObject.RepositoryID, infos[i].lfsMetaObject.Pointer)
		if err != nil {
			// OK Now we need to cleanup
			return cleanUpAfterFailure(ctx, &infos, t, err)
		}
		// Don't move the files yet - we need to ensure that
		// everything can be inserted first
	}

	// OK now we can insert the data into the store - there's no way to clean up the store
	// once it's in there, it's in there.
	contentStore := lfs.NewContentStore()
	for _, info := range infos {
		if err := uploadToLFSContentStore(info, contentStore); err != nil {
			return cleanUpAfterFailure(ctx, &infos, t, err)
		}
	}

	// Then push this tree to NewBranch
	if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
		return err
	}

	return repo_model.DeleteUploads(ctx, uploads...)
}

// From forgejo/services/repository/generate.go (but now allows /)
var fileNameSanitizeRegexp = regexp.MustCompile(`(?i)\.\.|[<>:\"\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`)

// Sanitize user input to valid OS filenames
//
//	       Based on https://github.com/sindresorhus/filename-reserved-regex
//	Adds ".." to prevent directory traversal
func fileNameSanitize(s string) string {
	// Added this because I am not sure what Windows will deliver us \ or / but we need /.
	s = strings.ReplaceAll(s, "\\", "/")
	return strings.TrimSpace(fileNameSanitizeRegexp.ReplaceAllString(s, "_"))
}

func copyUploadedLFSFilesIntoRepository(infos []uploadInfo, t *TemporaryUploadRepository, treePath string) error {
	var storeInLFSFunc func(string) (bool, error)

	if setting.LFS.StartServer {
		checker, err := t.gitRepo.GitAttributeChecker("", "filter")
		if err != nil {
			return err
		}
		defer checker.Close()

		storeInLFSFunc = func(name string) (bool, error) {
			attrs, err := checker.CheckPath(name)
			if err != nil {
				return false, fmt.Errorf("could not CheckPath(%s): %w", name, err)
			}
			return attrs["filter"] == "lfs", nil
		}
	}

	// Copy uploaded files into repository.
	for i, info := range infos {
		storeInLFS := false
		if storeInLFSFunc != nil {
			var err error
			storeInLFS, err = storeInLFSFunc(info.upload.Name)
			if err != nil {
				return err
			}
		}

		if err := copyUploadedLFSFileIntoRepository(&infos[i], storeInLFS, t, treePath); err != nil {
			return err
		}
	}
	return nil
}

func copyUploadedLFSFileIntoRepository(info *uploadInfo, storeInLFS bool, t *TemporaryUploadRepository, treePath string) error {
	file, err := os.Open(info.upload.LocalPath())
	if err != nil {
		return err
	}
	defer file.Close()

	var objectHash string
	if storeInLFS {
		// Handle LFS
		// FIXME: Inefficient! this should probably happen in models.Upload
		pointer, err := lfs.GeneratePointer(file)
		if err != nil {
			return err
		}

		info.lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID}

		if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil {
			return err
		}
	} else if objectHash, err = t.HashObject(file); err != nil {
		return err
	}

	// Add the object to the index
	return t.AddObjectToIndex("100644", objectHash, path.Join(treePath, info.upload.Name))
}

func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) error {
	if info.lfsMetaObject == nil {
		return nil
	}
	exist, err := contentStore.Exists(info.lfsMetaObject.Pointer)
	if err != nil {
		return err
	}
	if !exist {
		file, err := os.Open(info.upload.LocalPath())
		if err != nil {
			return err
		}

		defer file.Close()
		// FIXME: Put regenerates the hash and copies the file over.
		// I guess this strictly ensures the soundness of the store but this is inefficient.
		if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil {
			// OK Now we need to cleanup
			// Can't clean up the store, once uploaded there they're there.
			return err
		}
	}
	return nil
}

func copyUploadedFilesIntoAnnexRepository(infos []uploadInfo, t *TemporaryUploadRepository, treePath string) error {
	for i := range len(infos) {
		if err := copyUploadedFileIntoAnnexRepository(&infos[i], t, treePath); err != nil {
			return err
		}
	}
	return nil
}

func copyUploadedFileIntoAnnexRepository(info *uploadInfo, t *TemporaryUploadRepository, treePath string) error {
	pathInRepo := path.Join(t.basePath, treePath, info.upload.Name)
	if err := os.MkdirAll(filepath.Dir(pathInRepo), 0o700); err != nil {
		return err
	}
	if err := os.Rename(info.upload.LocalPath(), pathInRepo); err != nil {
		// Rename didn't work, try copy and remove
		inputFile, err := os.Open(info.upload.LocalPath())
		if err != nil {
			return fmt.Errorf("could not open source file: %v", err)
		}
		defer inputFile.Close()
		outputFile, err := os.Create(pathInRepo)
		if err != nil {
			return fmt.Errorf("could not open dest file: %v", err)
		}
		defer outputFile.Close()
		_, err = io.Copy(outputFile, inputFile)
		if err != nil {
			return fmt.Errorf("could not copy to dest from source: %v", err)
		}
		inputFile.Close()
		err = os.Remove(info.upload.LocalPath())
		if err != nil {
			return fmt.Errorf("could not remove source file: %v", err)
		}
	}
	return t.AddAnnex(pathInRepo)
}

func moveAnnexedFilesToOrigin(t *TemporaryUploadRepository, author, committer *user_model.User) error {
	authorSig := author.NewGitSig()
	committerSig := committer.NewGitSig()
	env := append(os.Environ(),
		"GIT_AUTHOR_NAME="+authorSig.Name,
		"GIT_AUTHOR_EMAIL="+authorSig.Email,
		"GIT_COMMITTER_NAME="+committerSig.Name,
		"GIT_COMMITTER_EMAIL="+committerSig.Email,
	)
	if _, _, err := git.NewCommand(t.ctx, "annex", "move", "--to", "origin").RunStdString(&git.RunOpts{Dir: t.basePath, Env: env}); err != nil {
		return err
	}
	return nil
}