mirror of
https://github.com/navidrome/navidrome.git
synced 2026-11-14 06:08:46 +01:00
fix(artwork): convert animated GIF covers in the background
Resized animated GIF covers were converted to animated WebP by ffmpeg inside the image request, bound to its 10s timeout. On slow CPUs large GIFs never finished: ffmpeg was killed, nothing was cached, and every request repeated the work. With ffmpeg 5.1 the conversion also failed mid-stream when reading the GIF from a pipe, after the stream was already handed to the client, so there was no fallback. Requests now get a static stand-in right away, served with no-store and no validators, while the conversion runs in the background. A single process-wide slot bounds these conversions, since they outlive the request throttle; when it is busy nothing is queued and a later request retries. The result, or a static fallback if ffmpeg fails, is cached under the same key. ffmpeg output is now buffered so mid-stream failures fall back to a static resize. The file cache gains cache.Transient for loader results that must be served but not stored. Without an available cache, conversion stays inline.
This commit is contained in:
parent
dd71f1c57f
commit
bc16cafb91
9 changed files with 362 additions and 9 deletions
36
utils/cache/file_caches.go
vendored
36
utils/cache/file_caches.go
vendored
|
|
@ -25,6 +25,18 @@ type Item interface {
|
|||
Key() string
|
||||
}
|
||||
|
||||
// Transient wraps a ReadFunc result that Get must serve but never store.
|
||||
func Transient(rc io.ReadCloser) io.ReadCloser { return transientReader{rc} }
|
||||
|
||||
type transientReader struct{ io.ReadCloser }
|
||||
|
||||
func unwrapTransient(r io.Reader) (io.Reader, bool) {
|
||||
if t, ok := r.(transientReader); ok {
|
||||
return t.ReadCloser, true
|
||||
}
|
||||
return r, false
|
||||
}
|
||||
|
||||
// ReadFunc is a function that retrieves the data to be cached. It receives the Item to be cached and returns
|
||||
// an io.Reader with the data and an error.
|
||||
type ReadFunc func(ctx context.Context, item Item) (io.Reader, error)
|
||||
|
|
@ -155,7 +167,8 @@ func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CachedStream{Reader: reader}, nil
|
||||
reader, transient := unwrapTransient(reader)
|
||||
return &CachedStream{Reader: reader, Transient: transient}, nil
|
||||
}
|
||||
|
||||
key := arg.Key()
|
||||
|
|
@ -182,6 +195,10 @@ func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
|
|||
_ = fc.invalidate(ctx, key)
|
||||
return nil, err
|
||||
}
|
||||
if reader, transient := unwrapTransient(reader); transient {
|
||||
fc.discard(ctx, key, r, w)
|
||||
return &CachedStream{Reader: reader, Transient: true}, nil
|
||||
}
|
||||
go func() {
|
||||
if err := fc.copyAndClose(ctx, key, w, reader); err != nil {
|
||||
log.Debug(ctx, "Error storing file in cache", "cache", fc.name, "key", key, err)
|
||||
|
|
@ -213,12 +230,27 @@ func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
|
|||
return &CachedStream{Reader: r, Cached: cached}, nil
|
||||
}
|
||||
|
||||
// discard drops a miss entry that will never be written. Cancelling first makes the removal
|
||||
// non-blocking, so the key is free when Get returns; readers that joined meanwhile fail.
|
||||
func (fc *fileCache) discard(ctx context.Context, key string, r io.Closer, w io.WriteCloser) {
|
||||
_ = r.Close()
|
||||
if cw, ok := w.(interface{ CloseWithError(error) error }); ok {
|
||||
_ = cw.CloseWithError(errTransientEntry)
|
||||
} else {
|
||||
_ = w.Close()
|
||||
}
|
||||
_ = fc.invalidate(ctx, key)
|
||||
}
|
||||
|
||||
var errTransientEntry = errors.New("cache entry was served without being stored")
|
||||
|
||||
// CachedStream is a wrapper around an io.ReadCloser that allows reading from a cache.
|
||||
type CachedStream struct {
|
||||
io.Reader
|
||||
io.Seeker
|
||||
io.Closer
|
||||
Cached bool
|
||||
Cached bool
|
||||
Transient bool
|
||||
}
|
||||
|
||||
func (s *CachedStream) Close() error {
|
||||
|
|
|
|||
61
utils/cache/file_caches_test.go
vendored
61
utils/cache/file_caches_test.go
vendored
|
|
@ -106,6 +106,67 @@ var _ = Describe("File Caches", func() {
|
|||
Expect(called).To(BeTrue())
|
||||
})
|
||||
|
||||
It("serves a transient result without storing it", func() {
|
||||
var calls atomic.Int32
|
||||
fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
|
||||
calls.Add(1)
|
||||
return Transient(io.NopCloser(strings.NewReader("stand-in"))), nil
|
||||
})
|
||||
|
||||
for range 2 {
|
||||
s, err := fc.Get(context.Background(), &testArg{"transient"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(s.Transient).To(BeTrue())
|
||||
Expect(s.Cached).To(BeFalse())
|
||||
Expect(io.ReadAll(s)).To(Equal([]byte("stand-in")))
|
||||
Expect(s.Close()).To(Succeed())
|
||||
}
|
||||
Expect(calls.Load()).To(Equal(int32(2)))
|
||||
|
||||
dataPath := fcSpreadFS(fc).KeyMapper((&testArg{"transient"}).Key())
|
||||
_, statErr := os.Stat(dataPath + ".complete")
|
||||
Expect(os.IsNotExist(statErr)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("stores the next result once a transient one was served", func() {
|
||||
var calls atomic.Int32
|
||||
fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
return Transient(io.NopCloser(strings.NewReader("stand-in"))), nil
|
||||
}
|
||||
return strings.NewReader("real"), nil
|
||||
})
|
||||
|
||||
s, err := fc.Get(context.Background(), &testArg{"k"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(io.ReadAll(s)).To(Equal([]byte("stand-in")))
|
||||
_ = s.Close()
|
||||
|
||||
// The transient entry is gone when Get returns, so this is a MISS that stores.
|
||||
s, err = fc.Get(context.Background(), &testArg{"k"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(s.Transient).To(BeFalse())
|
||||
Expect(io.ReadAll(s)).To(Equal([]byte("real")))
|
||||
_ = s.Close()
|
||||
|
||||
Eventually(func() bool {
|
||||
s, _ := fc.Get(context.Background(), &testArg{"k"})
|
||||
defer s.Close()
|
||||
return s.Cached
|
||||
}).Should(BeTrue())
|
||||
Expect(calls.Load()).To(Equal(int32(2)))
|
||||
})
|
||||
|
||||
It("reports a transient result when the cache is disabled", func() {
|
||||
fc := callNewFileCache("test", "0", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
|
||||
return Transient(io.NopCloser(strings.NewReader("stand-in"))), nil
|
||||
})
|
||||
s, err := fc.Get(context.Background(), &testArg{"k"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(s.Transient).To(BeTrue())
|
||||
Expect(io.ReadAll(s)).To(Equal([]byte("stand-in")))
|
||||
})
|
||||
|
||||
It("writes a completion marker after a successful cache write", func() {
|
||||
fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
|
||||
return strings.NewReader("complete-data"), nil
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue