Refactor editor upload, update and delete to use git plumbing and add LFS support (#5702)

* Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFile * Use git plumbing for upload: #5621 repo_editor.go: GetDiffPreview * Use git plumbing for upload: #5621 repo_editor.go: DeleteRepoFile * Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFiles * Move branch checkout functions out of repo_editor.go as they are no longer used there * BUGFIX: The default permissions should be 100644 This is a change from the previous code but is more in keeping with the default behaviour of git. Signed-off-by: Andrew Thornton <art27@cantab.net> * Standardise cleanUploadFilename to more closely match git See verify_path in: 7f4e641693/read-cache.c (L951) Signed-off-by: Andrew Thornton <art27@cantab.net> * Redirect on bad paths Signed-off-by: Andrew Thornton <art27@cantab.net> * Refactor to move the uploading functions out to a module Signed-off-by: Andrew Thornton <art27@cantab.net> * Add LFS support Signed-off-by: Andrew Thornton <art27@cantab.net> * Update upload.go attribution header Upload.go is essentially the remnants of repo_editor.go. The remaining code is essentially unchanged from the Gogs code, hence the Gogs attribution. * Delete upload files after session committed * Ensure that GIT_AUTHOR_NAME etc. are valid for git see #5774 Signed-off-by: Andrew Thornton <art27@cantab.net> * Add in test cases per @lafriks comment * Add space between gitea and github imports Signed-off-by: Andrew Thornton <art27@cantab.net> * more examples in TestCleanUploadName Signed-off-by: Andrew Thornton <art27@cantab.net> * fix formatting Signed-off-by: Andrew Thornton <art27@cantab.net> * Set the SSH_ORIGINAL_COMMAND to ensure hooks are run Signed-off-by: Andrew Thornton <art27@cantab.net> * Switch off SSH_ORIGINAL_COMMAND Signed-off-by: Andrew Thornton <art27@cantab.net>
2025-05-20 14:00:04 +02:00 · 2019-02-12 13:07:31 +00:00 · 2019-02-12 13:07:31 +00:00 · 296814e887
commit 296814e887
parent fc038caa69
11 changed files with 1135 additions and 598 deletions
--- a/modules/uploader/upload.go
+++ b/modules/uploader/upload.go
@ -0,0 +1,206 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uploader
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/setting"
+
+	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
+)
+
+// UploadRepoFileOptions contains the uploaded repository file options
+type UploadRepoFileOptions struct {
+	LastCommitID string
+	OldBranch    string
+	NewBranch    string
+	TreePath     string
+	Message      string
+	Files        []string // In UUID format.
+}
+
+type uploadInfo struct {
+	upload        *models.Upload
+	lfsMetaObject *models.LFSMetaObject
+}
+
+func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error {
+	for _, info := range *infos {
+		if info.lfsMetaObject == nil {
+			continue
+		}
+		if !info.lfsMetaObject.Existing {
+			if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil {
+				original = fmt.Errorf("%v, %v", original, err)
+			}
+		}
+	}
+	return original
+}
+
+// UploadRepoFiles uploads files to the given repository
+func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error {
+	if len(opts.Files) == 0 {
+		return nil
+	}
+
+	uploads, err := models.GetUploadsByUUIDs(opts.Files)
+	if err != nil {
+		return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err)
+	}
+
+	t, err := NewTemporaryUploadRepository(repo)
+	defer t.Close()
+	if err != nil {
+		return err
+	}
+	if err := t.Clone(opts.OldBranch); err != nil {
+		return err
+	}
+	if err := t.SetDefaultIndex(); err != nil {
+		return err
+	}
+
+	names := make([]string, len(uploads))
+	infos := make([]uploadInfo, len(uploads))
+	for i, upload := range uploads {
+		names[i] = upload.Name
+		infos[i] = uploadInfo{upload: upload}
+	}
+
+	filename2attribute2info, err := t.CheckAttribute("filter", names...)
+	if err != nil {
+		return err
+	}
+
+	// Copy uploaded files into repository.
+	for i, uploadInfo := range infos {
+		file, err := os.Open(uploadInfo.upload.LocalPath())
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		var objectHash string
+		if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" {
+			// Handle LFS
+			// FIXME: Inefficient! this should probably happen in models.Upload
+			oid, err := models.GenerateLFSOid(file)
+			if err != nil {
+				return err
+			}
+			fileInfo, err := file.Stat()
+			if err != nil {
+				return err
+			}
+
+			uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID}
+
+			if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil {
+				return err
+			}
+			infos[i] = uploadInfo
+
+		} else {
+			if objectHash, err = t.HashObject(file); err != nil {
+				return err
+			}
+		}
+
+		// Add the object to the index
+		if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil {
+			return err
+
+		}
+	}
+
+	// Now write the tree
+	treeHash, err := t.WriteTree()
+	if err != nil {
+		return err
+	}
+
+	// Now commit the tree
+	commitHash, err := t.CommitTree(doer, treeHash, opts.Message)
+	if err != nil {
+		return err
+	}
+
+	// Now deal with LFS objects
+	for _, uploadInfo := range infos {
+		if uploadInfo.lfsMetaObject == nil {
+			continue
+		}
+		uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject)
+		if err != nil {
+			// OK Now we need to cleanup
+			return cleanUpAfterFailure(&infos, t, err)
+		}
+		// Don't move the files yet - we need to ensure that
+		// everything can be inserted first
+	}
+
+	// OK now we can insert the data into the store - there's no way to clean up the store
+	// once it's in there, it's in there.
+	contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
+	for _, uploadInfo := range infos {
+		if uploadInfo.lfsMetaObject == nil {
+			continue
+		}
+		if !contentStore.Exists(uploadInfo.lfsMetaObject) {
+			file, err := os.Open(uploadInfo.upload.LocalPath())
+			if err != nil {
+				return cleanUpAfterFailure(&infos, t, err)
+			}
+			defer file.Close()
+			// FIXME: Put regenerates the hash and copies the file over.
+			// I guess this strictly ensures the soundness of the store but this is inefficient.
+			if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil {
+				// OK Now we need to cleanup
+				// Can't clean up the store, once uploaded there they're there.
+				return cleanUpAfterFailure(&infos, t, err)
+			}
+		}
+	}
+
+	// Then push this tree to NewBranch
+	if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
+		return err
+	}
+
+	// Simulate push event.
+	oldCommitID := opts.LastCommitID
+	if opts.NewBranch != opts.OldBranch {
+		oldCommitID = git.EmptySHA
+	}
+
+	if err = repo.GetOwner(); err != nil {
+		return fmt.Errorf("GetOwner: %v", err)
+	}
+	err = models.PushUpdate(
+		opts.NewBranch,
+		models.PushUpdateOptions{
+			PusherID:     doer.ID,
+			PusherName:   doer.Name,
+			RepoUserName: repo.Owner.Name,
+			RepoName:     repo.Name,
+			RefFullName:  git.BranchPrefix + opts.NewBranch,
+			OldCommitID:  oldCommitID,
+			NewCommitID:  commitHash,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("PushUpdate: %v", err)
+	}
+	// FIXME: Should we models.UpdateRepoIndexer(repo) here?
+
+	return models.DeleteUploads(uploads...)
+}