Skip to content

Commit a9400ba

Browse files
jolheiserKN4CK3R
jolheiser
and
KN4CK3R
authored
Fix container blob mount (#22226) (#22476)
Backport #22226 Co-authored-by: KN4CK3R <admin@oldschoolhack.me>
1 parent 9a6d78e commit a9400ba

File tree

5 files changed

+120
-71
lines changed

5 files changed

+120
-71
lines changed

Diff for: ‎models/packages/container/search.go

+10
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type BlobSearchOptions struct {
2626
Digest string
2727
Tag string
2828
IsManifest bool
29+
Repository string
2930
}
3031

3132
func (opts *BlobSearchOptions) toConds() builder.Cond {
@@ -54,6 +55,15 @@ func (opts *BlobSearchOptions) toConds() builder.Cond {
5455

5556
cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
5657
}
58+
if opts.Repository != "" {
59+
var propsCond builder.Cond = builder.Eq{
60+
"package_property.ref_type": packages.PropertyTypePackage,
61+
"package_property.name": container_module.PropertyRepository,
62+
"package_property.value": opts.Repository,
63+
}
64+
65+
cond = cond.And(builder.In("package.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
66+
}
5767

5868
return cond
5969
}

Diff for: ‎routers/api/packages/container/blob.go

+76-53
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,60 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
3434

3535
contentStore := packages_module.NewContentStore()
3636

37+
uploadVersion, err := getOrCreateUploadVersion(pi)
38+
if err != nil {
39+
return nil, err
40+
}
41+
42+
err = db.WithTx(func(ctx context.Context) error {
43+
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
44+
if err != nil {
45+
log.Error("Error inserting package blob: %v", err)
46+
return err
47+
}
48+
// FIXME: Workaround to be removed in v1.20
49+
// https://github.com/go-gitea/gitea/issues/19586
50+
if exists {
51+
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
52+
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
53+
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
54+
exists = false
55+
}
56+
}
57+
if !exists {
58+
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
59+
log.Error("Error saving package blob in content store: %v", err)
60+
return err
61+
}
62+
}
63+
64+
return createFileForBlob(ctx, uploadVersion, pb)
65+
})
66+
if err != nil {
67+
if !exists {
68+
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
69+
log.Error("Error deleting package blob from content store: %v", err)
70+
}
71+
}
72+
return nil, err
73+
}
74+
75+
return pb, nil
76+
}
77+
78+
// mountBlob mounts the specific blob to a different package
79+
func mountBlob(pi *packages_service.PackageInfo, pb *packages_model.PackageBlob) error {
80+
uploadVersion, err := getOrCreateUploadVersion(pi)
81+
if err != nil {
82+
return err
83+
}
84+
85+
return db.WithTx(func(ctx context.Context) error {
86+
return createFileForBlob(ctx, uploadVersion, pb)
87+
})
88+
}
89+
90+
func getOrCreateUploadVersion(pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) {
3791
var uploadVersion *packages_model.PackageVersion
3892

3993
// FIXME: Replace usage of mutex with database transaction
@@ -84,66 +138,35 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
84138
return nil
85139
})
86140
uploadVersionMutex.Unlock()
87-
if err != nil {
88-
return nil, err
89-
}
90-
91-
err = db.WithTx(func(ctx context.Context) error {
92-
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
93-
if err != nil {
94-
log.Error("Error inserting package blob: %v", err)
95-
return err
96-
}
97-
// FIXME: Workaround to be removed in v1.20
98-
// https://github.com/go-gitea/gitea/issues/19586
99-
if exists {
100-
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
101-
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
102-
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
103-
exists = false
104-
}
105-
}
106-
if !exists {
107-
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
108-
log.Error("Error saving package blob in content store: %v", err)
109-
return err
110-
}
111-
}
112141

113-
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
142+
return uploadVersion, err
143+
}
114144

115-
pf := &packages_model.PackageFile{
116-
VersionID: uploadVersion.ID,
117-
BlobID: pb.ID,
118-
Name: filename,
119-
LowerName: filename,
120-
CompositeKey: packages_model.EmptyFileKey,
121-
}
122-
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
123-
if err == packages_model.ErrDuplicatePackageFile {
124-
return nil
125-
}
126-
log.Error("Error inserting package file: %v", err)
127-
return err
128-
}
145+
func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, pb *packages_model.PackageBlob) error {
146+
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
129147

130-
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil {
131-
log.Error("Error setting package file property: %v", err)
132-
return err
148+
pf := &packages_model.PackageFile{
149+
VersionID: pv.ID,
150+
BlobID: pb.ID,
151+
Name: filename,
152+
LowerName: filename,
153+
CompositeKey: packages_model.EmptyFileKey,
154+
}
155+
var err error
156+
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
157+
if err == packages_model.ErrDuplicatePackageFile {
158+
return nil
133159
}
160+
log.Error("Error inserting package file: %v", err)
161+
return err
162+
}
134163

135-
return nil
136-
})
137-
if err != nil {
138-
if !exists {
139-
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
140-
log.Error("Error deleting package blob from content store: %v", err)
141-
}
142-
}
143-
return nil, err
164+
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil {
165+
log.Error("Error setting package file property: %v", err)
166+
return err
144167
}
145168

146-
return pb, nil
169+
return nil
147170
}
148171

149172
func deleteBlob(ownerID int64, image, digest string) error {

Diff for: ‎routers/api/packages/container/container.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -196,10 +196,15 @@ func InitiateUploadBlob(ctx *context.Context) {
196196
from := ctx.FormTrim("from")
197197
if mount != "" {
198198
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
199-
Image: from,
200-
Digest: mount,
199+
Repository: from,
200+
Digest: mount,
201201
})
202202
if blob != nil {
203+
if err := mountBlob(&packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}, blob.Blob); err != nil {
204+
apiError(ctx, http.StatusInternalServerError, err)
205+
return
206+
}
207+
203208
setResponseHeaders(ctx.Resp, &containerHeaders{
204209
Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount),
205210
ContentDigest: mount,

Diff for: ‎templates/package/settings.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
{{.locale.Tr "packages.settings.delete"}}
5252
</div>
5353
<div class="content">
54-
<div class="ui warning message text left">
54+
<div class="ui warning message text left word-break">
5555
{{.locale.Tr "packages.settings.delete.notice" .PackageDescriptor.Package.Name .PackageDescriptor.Version.Version}}
5656
</div>
5757
<form class="ui form" action="{{.Link}}" method="post">

Diff for: ‎tests/integration/api_packages_container_test.go

+26-15
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,32 @@ func TestPackageContainer(t *testing.T) {
257257
})
258258
})
259259

260+
t.Run("UploadBlob/Mount", func(t *testing.T) {
261+
defer tests.PrintCurrentTest(t)()
262+
263+
req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
264+
addTokenAuthHeader(req, userToken)
265+
MakeRequest(t, req, http.StatusAccepted)
266+
267+
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
268+
addTokenAuthHeader(req, userToken)
269+
resp := MakeRequest(t, req, http.StatusCreated)
270+
271+
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
272+
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
273+
274+
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s", url, unknownDigest, "unknown/image"))
275+
addTokenAuthHeader(req, userToken)
276+
MakeRequest(t, req, http.StatusAccepted)
277+
278+
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s/%s", url, blobDigest, user.Name, image))
279+
addTokenAuthHeader(req, userToken)
280+
resp = MakeRequest(t, req, http.StatusCreated)
281+
282+
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
283+
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
284+
})
285+
260286
for _, tag := range tags {
261287
t.Run(fmt.Sprintf("[Tag:%s]", tag), func(t *testing.T) {
262288
t.Run("UploadManifest", func(t *testing.T) {
@@ -445,21 +471,6 @@ func TestPackageContainer(t *testing.T) {
445471
assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest))
446472
})
447473

448-
t.Run("UploadBlob/Mount", func(t *testing.T) {
449-
defer tests.PrintCurrentTest(t)()
450-
451-
req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
452-
addTokenAuthHeader(req, userToken)
453-
MakeRequest(t, req, http.StatusAccepted)
454-
455-
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
456-
addTokenAuthHeader(req, userToken)
457-
resp := MakeRequest(t, req, http.StatusCreated)
458-
459-
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
460-
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
461-
})
462-
463474
t.Run("HeadBlob", func(t *testing.T) {
464475
defer tests.PrintCurrentTest(t)()
465476

0 commit comments

Comments
 (0)