mirror of
https://github.com/navidrome/navidrome.git
synced 2026-11-21 17:40:28 +01:00
perf(artwork): cap the stale-absent recheck at 100 items per kind per hour (#6007)
* feat(artwork): drip the stale-absent recheck instead of bursting it daily Each hourly housekeeping tick now re-queues at most 100 absent states per kind, oldest attempts first, instead of everything older than 24h at once. External agents see a flat ~100 requests/hour per agent instead of hourly bursts of ~2,000, and the effective recheck interval self-scales with the size of the absent pool (~4 days at 10k absent artists) while small libraries keep the 24h floor. * feat(artwork): trust an absent artwork state for a week before rechecking With the recheck now dripped at 100 items per kind per hour, the 24h floor only governed small libraries, where the drip cap never binds; they still re-asked every agent daily. A 7-day floor cuts that cost 7x and, for large libraries, becomes the binding limit over the drip cycle (~5.7k calls/day instead of ~9.6k at 10k absent artists). Among comparable servers, this is still the second-most-eager recheck: gonic retries misses every 30 days, Jellyfin and Funkwhale never do. * refactor(artwork): state the drip's backpressure contract where it bites Review follow-ups: the recheck limit deliberately caps the *selection*, not the insertions — already-queued rows use up budget, so a stalled drain admits no new work instead of building a recovery burst. Say so in the interface doc, mirror it in the mock by truncating the sorted candidates (matching the SQL's LIMIT-before-ON CONFLICT), and teach `artwork status` and the worker doc the post-drip wording. Also pin the one cmd fixture that still assumed a 24h recheck window.
This commit is contained in:
parent
ffc68e29db
commit
07b6411c0b
9 changed files with 70 additions and 23 deletions
|
|
@ -56,11 +56,12 @@ func (r *artworkQueueRepository) EnqueuePreservingBackoff(items ...model.Artwork
|
|||
priority = MAX(priority, excluded.priority)`, items)
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time) (int64, error) {
|
||||
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time, limit int) (int64, error) {
|
||||
now := time.Now()
|
||||
return r.insertIfNotQueued("", `SELECT item_kind, item_id, image_type, ?, 0, ?, ?
|
||||
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?`,
|
||||
model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore)
|
||||
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
|
||||
ORDER BY attempted_at LIMIT ?`,
|
||||
model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore, limit)
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) EnqueueAllMissing(kind model.Kind, priority int) (int64, error) {
|
||||
|
|
@ -230,7 +231,7 @@ func (r *artworkQueueRepository) Count() (int64, error) {
|
|||
return res.Count, err
|
||||
}
|
||||
|
||||
// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is what a recheck would queue.
|
||||
// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is the pool a recheck drains from.
|
||||
func (r *artworkQueueRepository) CountAbsent(kind model.Kind, attemptedBefore time.Time) (model.ArtworkAbsentStat, error) {
|
||||
var res model.ArtworkAbsentStat
|
||||
err := r.queryOne(Select("count(*) as total").
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ var _ = Describe("ArtworkQueueRepository", func() {
|
|||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
|
||||
|
||||
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour))
|
||||
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour), 100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(int64(1)))
|
||||
|
||||
|
|
@ -255,6 +255,23 @@ var _ = Describe("ArtworkQueueRepository", func() {
|
|||
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
|
||||
})
|
||||
|
||||
It("enqueues only the oldest stale absent states up to the limit", func() {
|
||||
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||
now := time.Now()
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "oldest", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-72 * time.Hour)})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "older", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-60 * time.Hour)})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "old", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-48 * time.Hour)})).To(Succeed())
|
||||
|
||||
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, now.Add(-24*time.Hour), 2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(int64(2)))
|
||||
|
||||
items, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
ids := slice.Map(items, func(it model.ArtworkQueueItem) string { return it.ItemID })
|
||||
Expect(ids).To(ConsistOf("oldest", "older"))
|
||||
})
|
||||
|
||||
It("enqueues entities that have no item_artwork row at all", func() {
|
||||
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: time.Now()})).To(Succeed())
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue