feat: support grouping by any path for arch package (#4903)

Previous arch package grouping was not well-suited for complex or multi-architecture environments. It now supports the following content:

- Support grouping by any path.
- New support for packages in `xz` format.
- Fix clean up rules

<!--start release-notes-assistant-->

## Draft release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Features
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/4903): <!--number 4903 --><!--line 0 --><!--description c3VwcG9ydCBncm91cGluZyBieSBhbnkgcGF0aCBmb3IgYXJjaCBwYWNrYWdl-->support grouping by any path for arch package<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4903
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Exploding Dragon <explodingfkl@gmail.com>
Co-committed-by: Exploding Dragon <explodingfkl@gmail.com>
This commit is contained in:
Exploding Dragon 2024-08-11 10:35:11 +00:00 committed by Earl Warren
parent a4da672134
commit 87d50eca87
7 changed files with 309 additions and 218 deletions

View file

@ -81,18 +81,18 @@ jMBmtEhxyCnCZdUAwYKxAxeRFVk4TCL0aYgWjt3kHTg9SjVStppI2YCSWshUEFGdmJmyCVGpnqIU
KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL
TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`),
"other": unPack(`
KLUv/QBYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj
ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi
sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz
q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf
A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz
5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd
GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW
gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX
Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv
qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE
wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd
nYAR`),
/Td6WFoAAATm1rRGBMCyBIAYIQEWAAAAAAAAABaHRszgC/8CKl0AFxNGhTWwfXmuDQEJlHgNLrkq
VxpJY6d9iRTt6gB4uCj0481rnYfXaUADHzOFuF3490RPrM6juPXrknqtVyuWJ5efW19BgwctN6xk
UiXiZaXVAWVWJWy2XHJiyYCMWBfIjUfo1ccOgwolwgFHJ64ZJjbayA3k6lYPcImuAqYL5NEVHpwl
Z8CWIjiXXSMQGsB3gxMdq9nySZbHQLK/KCKQ+oseF6kXyIgSEyuG4HhjVBBYIwTvWzI06kjNUXEy
2sw0n50uocLSAwJ/3mdX3n3XF5nmmuQMPtFbdQgQtC2VhyVd3TdIF+pT6zAEzXFJJ3uLkNbKSS88
ZdBny6X/ftT5lQpNi/Wg0xLEQA4m4fu4fRAR0kOKzHM2svNLbTxa/wOPidqPzR6b/jfKmHkXxBNa
jFafty0a5K2S3F6JpwXZ2fqti/zG9NtMc+bbuXycC327EofXRXNtuOupELDD+ltTOIBF7CcTswyi
MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv
0u2hqt4NXPNYqaMm9bFnnIUxEN82HgNWBcXf2baWKOdGzPzCuWg2fAM4zxHnBWcimxLXiJgaI8mU
J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE
SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ
Wg==`), // this is tar.xz file
}
t.Run("RepositoryKey", func(t *testing.T) {
@ -105,155 +105,154 @@ nYAR`),
require.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----")
})
t.Run("Upload", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"]))
MakeRequest(t, req, http.StatusUnauthorized)
req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch)
require.NoError(t, err)
require.Len(t, pvs, 1)
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
require.NoError(t, err)
require.Nil(t, pd.SemVer)
require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata)
require.Equal(t, "test", pd.Package.Name)
require.Equal(t, "1.0.0-1", pd.Version.Version)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
require.NoError(t, err)
require.Len(t, pfs, 2) // zst and zst.sig
require.True(t, pfs[0].IsLead)
pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID)
require.NoError(t, err)
require.Equal(t, int64(len(pkgs["any"])), pb.Size)
req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusConflict)
req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["x86_64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["other"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["x86_64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
})
t.Run("Download", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst")
resp := MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["x86_64"], resp.Body.Bytes())
req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-any.pkg.tar.zst")
resp = MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["any"], resp.Body.Bytes())
req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst")
MakeRequest(t, req, http.StatusNotFound)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst")
MakeRequest(t, req, http.StatusNotFound)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst")
resp = MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["any"], resp.Body.Bytes())
})
t.Run("SignVerify", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig")
respSig := MakeRequest(t, req, http.StatusOK)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
for _, group := range []string{"", "arch", "arch/os", "x86_64"} {
groupURL := rootURL
if group != "" {
groupURL = groupURL + "/" + group
}
})
t.Run(fmt.Sprintf("Upload[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
t.Run("Repository", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"]))
MakeRequest(t, req, http.StatusUnauthorized)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db.sig")
respSig := MakeRequest(t, req, http.StatusOK)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewBuffer([]byte("any string"))).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusBadRequest)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
}
files, err := listGzipFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 2)
for s, d := range files {
name := getProperty(string(d.Data), "NAME")
ver := getProperty(string(d.Data), "VERSION")
require.Equal(t, name+"-"+ver+"/desc", s)
fn := getProperty(string(d.Data), "FILENAME")
pgp := getProperty(string(d.Data), "PGPSIG")
req = NewRequest(t, "GET", rootURL+"/base/x86_64/"+fn+".sig")
respSig := MakeRequest(t, req, http.StatusOK)
decodeString, err := base64.StdEncoding.DecodeString(pgp)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch)
require.NoError(t, err)
require.Equal(t, respSig.Body.Bytes(), decodeString)
}
})
t.Run("Delete", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNotFound)
require.Len(t, pvs, 1)
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
require.NoError(t, err)
require.Nil(t, pd.SemVer)
require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata)
require.Equal(t, "test", pd.Package.Name)
require.Equal(t, "1.0.0-1", pd.Version.Version)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
files, err := listGzipFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
require.NoError(t, err)
size := 0
for _, pf := range pfs {
if pf.CompositeKey == group {
size++
}
}
require.Equal(t, 2, size) // zst and zst.sig
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test2/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db")
MakeRequest(t, req, http.StatusNotFound)
pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID)
require.NoError(t, err)
require.Equal(t, int64(len(pkgs["any"])), pb.Size)
req = NewRequest(t, "GET", rootURL+"/default/x86_64/base.db")
respPkg = MakeRequest(t, req, http.StatusOK)
files, err = listGzipFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1)
})
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name) // exists
MakeRequest(t, req, http.StatusConflict)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["x86_64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name) // exists again
MakeRequest(t, req, http.StatusConflict)
})
t.Run(fmt.Sprintf("Download[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst")
resp := MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["x86_64"], resp.Body.Bytes())
req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst")
resp = MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["any"], resp.Body.Bytes())
// get other group
req = NewRequest(t, "GET", rootURL+"/unknown/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst")
MakeRequest(t, req, http.StatusNotFound)
})
t.Run(fmt.Sprintf("SignVerify[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig")
respSig := MakeRequest(t, req, http.StatusOK)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
}
})
t.Run(fmt.Sprintf("RepositoryDB[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db.sig")
respSig := MakeRequest(t, req, http.StatusOK)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
}
files, err := listTarGzFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1)
for s, d := range files {
name := getProperty(string(d.Data), "NAME")
ver := getProperty(string(d.Data), "VERSION")
require.Equal(t, name+"-"+ver+"/desc", s)
fn := getProperty(string(d.Data), "FILENAME")
pgp := getProperty(string(d.Data), "PGPSIG")
req = NewRequest(t, "GET", groupURL+"/x86_64/"+fn+".sig")
respSig := MakeRequest(t, req, http.StatusOK)
decodeString, err := base64.StdEncoding.DecodeString(pgp)
require.NoError(t, err)
require.Equal(t, respSig.Body.Bytes(), decodeString)
}
})
t.Run(fmt.Sprintf("Delete[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// test data
req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["other"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNotFound)
req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
files, err := listTarGzFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1) // other pkg in L225
req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
MakeRequest(t, req, http.StatusNotFound)
})
}
}
func getProperty(data, key string) string {
@ -270,7 +269,7 @@ func getProperty(data, key string) string {
}
}
func listGzipFiles(data []byte) (fstest.MapFS, error) {
func listTarGzFiles(data []byte) (fstest.MapFS, error) {
reader, err := gzip.NewReader(bytes.NewBuffer(data))
defer reader.Close()
if err != nil {