mirror of
https://codeberg.org/davrot/forgejo.git
synced 2025-05-31 12:00:01 +02:00
Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)
* Dump github/gitlab repository data to a local directory * Fix lint * Adjust directory structure * Allow migration special units * Allow migration ignore release assets * Fix lint * Add restore repository * stage the changes * Merge * Fix lint * Update the interface * Add some restore methods * Finish restore * Add comments * Fix restore * Add a token flag * Fix bug * Fix test * Fix test * Fix bug * Fix bug * Fix lint * Fix restore * refactor downloader * fmt * Fix bug isEnd detection on getIssues * Refactor maxPerPage * Remove unused codes * Remove unused codes * Fix bug * Fix restore * Fix dump * Uploader should not depend downloader * use release attachment name but not id * Fix restore bug * Fix lint * Fix restore bug * Add a method of DownloadFunc for base.Release to make uploader not depend on downloader * fix Release yml marshal * Fix trace information * Fix bug when dump & restore * Save relative path on yml file * Fix bug * Use relative path * Update docs * Use git service string but not int * Recognize clone addr to service type
This commit is contained in:
Lunny Xiao
GitHub
parent
212fa340cf
commit
dd08853b10
29 changed files with 1491 additions and 232 deletions
276
modules/migrations/restore.go
Normal file
276
modules/migrations/restore.go
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// RepositoryRestorer implements an Downloader from the local directory
|
||||
type RepositoryRestorer struct {
|
||||
ctx context.Context
|
||||
baseDir string
|
||||
repoOwner string
|
||||
repoName string
|
||||
}
|
||||
|
||||
// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
|
||||
func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
|
||||
baseDir, err := filepath.Abs(baseDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &RepositoryRestorer{
|
||||
ctx: ctx,
|
||||
baseDir: baseDir,
|
||||
repoOwner: owner,
|
||||
repoName: repoName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *RepositoryRestorer) commentDir() string {
|
||||
return filepath.Join(r.baseDir, "comments")
|
||||
}
|
||||
|
||||
func (r *RepositoryRestorer) reviewDir() string {
|
||||
return filepath.Join(r.baseDir, "reviews")
|
||||
}
|
||||
|
||||
// SetContext set context
|
||||
func (r *RepositoryRestorer) SetContext(ctx context.Context) {
|
||||
r.ctx = ctx
|
||||
}
|
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
|
||||
p := filepath.Join(r.baseDir, "repo.yml")
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var opts = make(map[string]string)
|
||||
err = yaml.Unmarshal(bs, &opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isPrivate, _ := strconv.ParseBool(opts["is_private"])
|
||||
|
||||
return &base.Repository{
|
||||
Owner: r.repoOwner,
|
||||
Name: r.repoName,
|
||||
IsPrivate: isPrivate,
|
||||
Description: opts["description"],
|
||||
OriginalURL: opts["original_url"],
|
||||
CloneURL: opts["clone_addr"],
|
||||
DefaultBranch: opts["default_branch"],
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetTopics return github topics
|
||||
func (r *RepositoryRestorer) GetTopics() ([]string, error) {
|
||||
p := filepath.Join(r.baseDir, "topic.yml")
|
||||
|
||||
var topics = struct {
|
||||
Topics []string `yaml:"topics"`
|
||||
}{}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &topics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return topics.Topics, nil
|
||||
}
|
||||
|
||||
// GetMilestones returns milestones
|
||||
func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
|
||||
var milestones = make([]*base.Milestone, 0, 10)
|
||||
p := filepath.Join(r.baseDir, "milestone.yml")
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &milestones)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return milestones, nil
|
||||
}
|
||||
|
||||
// GetReleases returns releases
|
||||
func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
|
||||
var releases = make([]*base.Release, 0, 10)
|
||||
p := filepath.Join(r.baseDir, "release.yml")
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &releases)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rel := range releases {
|
||||
for _, asset := range rel.Assets {
|
||||
*asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
|
||||
}
|
||||
}
|
||||
return releases, nil
|
||||
}
|
||||
|
||||
// GetLabels returns labels
|
||||
func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
|
||||
var labels = make([]*base.Label, 0, 10)
|
||||
p := filepath.Join(r.baseDir, "label.yml")
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &labels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
// GetIssues returns issues according start and limit
|
||||
func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
|
||||
var issues = make([]*base.Issue, 0, 10)
|
||||
p := filepath.Join(r.baseDir, "issue.yml")
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, true, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &issues)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return issues, true, nil
|
||||
}
|
||||
|
||||
// GetComments returns comments according issueNumber
|
||||
func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) {
|
||||
var comments = make([]*base.Comment, 0, 10)
|
||||
p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber))
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &comments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return comments, nil
|
||||
}
|
||||
|
||||
// GetPullRequests returns pull requests according page and perPage
|
||||
func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
|
||||
var pulls = make([]*base.PullRequest, 0, 10)
|
||||
p := filepath.Join(r.baseDir, "pull_request.yml")
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, true, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &pulls)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
for _, pr := range pulls {
|
||||
pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
|
||||
}
|
||||
return pulls, true, nil
|
||||
}
|
||||
|
||||
// GetReviews returns pull requests review
|
||||
func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
|
||||
var reviews = make([]*base.Review, 0, 10)
|
||||
p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber))
|
||||
_, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(bs, &reviews)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reviews, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue