Avatars and Repo avatars support storing in minio (#12516)

* Avatar support minio

* Support repo avatar minio storage

* Add missing migration

* Fix bug

* Fix test

* Add test for minio store type on avatars and repo avatars; Add documents

* Fix bug

* Fix bug

* Add back missed avatar link method

* refactor codes

* Simplify the codes

* Code improvements

* Fix lint

* Fix test mysql

* Fix test mysql

* Fix test mysql

* Fix settings

* Fix test

* fix test

* Fix bug
This commit is contained in:
Lunny Xiao 2020-10-14 21:07:51 +08:00 committed by GitHub
parent 93f7525061
commit 80a6b0f5bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 705 additions and 477 deletions

View file

@ -8,29 +8,26 @@ package models
import (
"container/list"
"context"
"crypto/md5"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"errors"
"fmt"
_ "image/jpeg" // Needed for jpeg support
"image/png"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"unicode/utf8"
"code.gitea.io/gitea/modules/avatar"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/public"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
@ -347,104 +344,6 @@ func (u *User) GenerateActivateCode() string {
return u.GenerateEmailActivateCode(u.Email)
}
// CustomAvatarPath returns user custom avatar file path.
func (u *User) CustomAvatarPath() string {
return filepath.Join(setting.AvatarUploadPath, u.Avatar)
}
// GenerateRandomAvatar generates a random avatar for user.
func (u *User) GenerateRandomAvatar() error {
return u.generateRandomAvatar(x)
}
func (u *User) generateRandomAvatar(e Engine) error {
seed := u.Email
if len(seed) == 0 {
seed = u.Name
}
img, err := avatar.RandomImage([]byte(seed))
if err != nil {
return fmt.Errorf("RandomImage: %v", err)
}
// NOTICE for random avatar, it still uses id as avatar name, but custom avatar use md5
// since random image is not a user's photo, there is no security for enumable
if u.Avatar == "" {
u.Avatar = fmt.Sprintf("%d", u.ID)
}
if err = os.MkdirAll(filepath.Dir(u.CustomAvatarPath()), os.ModePerm); err != nil {
return fmt.Errorf("MkdirAll: %v", err)
}
fw, err := os.Create(u.CustomAvatarPath())
if err != nil {
return fmt.Errorf("Create: %v", err)
}
defer fw.Close()
if _, err := e.ID(u.ID).Cols("avatar").Update(u); err != nil {
return err
}
if err = png.Encode(fw, img); err != nil {
return fmt.Errorf("Encode: %v", err)
}
log.Info("New random avatar created: %d", u.ID)
return nil
}
// SizedRelAvatarLink returns a link to the user's avatar via
// the local explore page. Function returns immediately.
// When applicable, the link is for an avatar of the indicated size (in pixels).
func (u *User) SizedRelAvatarLink(size int) string {
return strings.TrimSuffix(setting.AppSubURL, "/") + "/user/avatar/" + u.Name + "/" + strconv.Itoa(size)
}
// RealSizedAvatarLink returns a link to the user's avatar. When
// applicable, the link is for an avatar of the indicated size (in pixels).
//
// This function make take time to return when federated avatars
// are in use, due to a DNS lookup need
//
func (u *User) RealSizedAvatarLink(size int) string {
if u.ID == -1 {
return base.DefaultAvatarLink()
}
switch {
case u.UseCustomAvatar:
if !com.IsFile(u.CustomAvatarPath()) {
return base.DefaultAvatarLink()
}
return setting.AppSubURL + "/avatars/" + u.Avatar
case setting.DisableGravatar, setting.OfflineMode:
if !com.IsFile(u.CustomAvatarPath()) {
if err := u.GenerateRandomAvatar(); err != nil {
log.Error("GenerateRandomAvatar: %v", err)
}
}
return setting.AppSubURL + "/avatars/" + u.Avatar
}
return base.SizedAvatarLink(u.AvatarEmail, size)
}
// RelAvatarLink returns a relative link to the user's avatar. The link
// may either be a sub-URL to this site, or a full URL to an external avatar
// service.
func (u *User) RelAvatarLink() string {
return u.SizedRelAvatarLink(base.DefaultAvatarSize)
}
// AvatarLink returns user avatar absolute link.
func (u *User) AvatarLink() string {
link := u.RelAvatarLink()
if link[0] == '/' && link[1] != '/' {
return setting.AppURL + strings.TrimPrefix(link, setting.AppSubURL)[1:]
}
return link
}
// GetFollowers returns range of user's followers.
func (u *User) GetFollowers(listOptions ListOptions) ([]*User, error) {
sess := x.
@ -537,64 +436,6 @@ func (u *User) IsPasswordSet() bool {
return !u.ValidatePassword("")
}
// UploadAvatar saves custom avatar for user.
// FIXME: split uploads to different subdirs in case we have massive users.
func (u *User) UploadAvatar(data []byte) error {
m, err := avatar.Prepare(data)
if err != nil {
return err
}
sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
return err
}
u.UseCustomAvatar = true
// Different users can upload same image as avatar
// If we prefix it with u.ID, it will be separated
// Otherwise, if any of the users delete his avatar
// Other users will lose their avatars too.
u.Avatar = fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%x", u.ID, md5.Sum(data)))))
if err = updateUser(sess, u); err != nil {
return fmt.Errorf("updateUser: %v", err)
}
if err := os.MkdirAll(setting.AvatarUploadPath, os.ModePerm); err != nil {
return fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err)
}
fw, err := os.Create(u.CustomAvatarPath())
if err != nil {
return fmt.Errorf("Create: %v", err)
}
defer fw.Close()
if err = png.Encode(fw, *m); err != nil {
return fmt.Errorf("Encode: %v", err)
}
return sess.Commit()
}
// DeleteAvatar deletes the user's custom avatar.
func (u *User) DeleteAvatar() error {
log.Trace("DeleteAvatar[%d]: %s", u.ID, u.CustomAvatarPath())
if len(u.Avatar) > 0 {
if err := util.Remove(u.CustomAvatarPath()); err != nil {
return fmt.Errorf("Failed to remove %s: %v", u.CustomAvatarPath(), err)
}
}
u.UseCustomAvatar = false
u.Avatar = ""
if _, err := x.ID(u.ID).Cols("avatar, use_custom_avatar").Update(u); err != nil {
return fmt.Errorf("UpdateUser: %v", err)
}
return nil
}
// IsOrganization returns true if user is actually a organization.
func (u *User) IsOrganization() bool {
return u.Type == UserTypeOrganization
@ -1285,17 +1126,14 @@ func deleteUser(e *xorm.Session, u *User) error {
// Note: There are something just cannot be roll back,
// so just keep error logs of those operations.
path := UserPath(u.Name)
if err := util.RemoveAll(path); err != nil {
return fmt.Errorf("Failed to RemoveAll %s: %v", path, err)
}
if len(u.Avatar) > 0 {
avatarPath := u.CustomAvatarPath()
if com.IsExist(avatarPath) {
if err := util.Remove(avatarPath); err != nil {
return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
}
avatarPath := u.CustomAvatarRelativePath()
if err := storage.Avatars.Delete(avatarPath); err != nil {
return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
}
}
@ -2034,3 +1872,25 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error {
}
return nil
}
// IterateUser iterate users
func IterateUser(f func(user *User) error) error {
var start int
var batchSize = setting.Database.IterateBufferSize
for {
var users = make([]*User, 0, batchSize)
if err := x.Limit(batchSize, start).Find(&users); err != nil {
return err
}
if len(users) == 0 {
return nil
}
start += len(users)
for _, user := range users {
if err := f(user); err != nil {
return err
}
}
}
}