perf(artwork): cap the stale-absent recheck at 100 items per kind per hour (#6007)

* feat(artwork): drip the stale-absent recheck instead of bursting it daily

Each hourly housekeeping tick now re-queues at most 100 absent states
per kind, oldest attempts first, instead of everything older than 24h
at once. External agents see a flat ~100 requests/hour per agent
instead of hourly bursts of ~2,000, and the effective recheck interval
self-scales with the size of the absent pool (~4 days at 10k absent
artists) while small libraries keep the 24h floor.

* feat(artwork): trust an absent artwork state for a week before rechecking

With the recheck now dripped at 100 items per kind per hour, the 24h
floor only governed small libraries, where the drip cap never binds;
they still re-asked every agent daily. A 7-day floor cuts that cost 7x
and, for large libraries, becomes the binding limit over the drip
cycle (~5.7k calls/day instead of ~9.6k at 10k absent artists).

Among comparable servers, this is still the second-most-eager recheck:
gonic retries misses every 30 days, Jellyfin and Funkwhale never do.

* refactor(artwork): state the drip's backpressure contract where it bites

Review follow-ups: the recheck limit deliberately caps the *selection*,
not the insertions — already-queued rows use up budget, so a stalled
drain admits no new work instead of building a recovery burst. Say so
in the interface doc, mirror it in the mock by truncating the sorted
candidates (matching the SQL's LIMIT-before-ON CONFLICT), and teach
`artwork status` and the worker doc the post-drip wording. Also pin
the one cmd fixture that still assumed a 24h recheck window.
This commit is contained in:
Deluan Quintão 2026-08-21 15:23:07 -04:00 • committed by GitHub
commit 07b6411c0b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 70 additions and 23 deletions

View file

@ -56,11 +56,12 @@ func (r *artworkQueueRepository) EnqueuePreservingBackoff(items ...model.Artwork
priority = MAX(priority, excluded.priority)`, items)
}
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time) (int64, error) {
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time, limit int) (int64, error) {
now := time.Now()
return r.insertIfNotQueued("", `SELECT item_kind, item_id, image_type, ?, 0, ?, ?
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?`,
model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore)
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
ORDER BY attempted_at LIMIT ?`,
model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore, limit)
}
func (r *artworkQueueRepository) EnqueueAllMissing(kind model.Kind, priority int) (int64, error) {
@ -230,7 +231,7 @@ func (r *artworkQueueRepository) Count() (int64, error) {
return res.Count, err
}
// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is what a recheck would queue.
// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is the pool a recheck drains from.
func (r *artworkQueueRepository) CountAbsent(kind model.Kind, attemptedBefore time.Time) (model.ArtworkAbsentStat, error) {
var res model.ArtworkAbsentStat
err := r.queryOne(Select("count(*) as total").

View file

@ -244,7 +244,7 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour))
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour), 100)
Expect(err).ToNot(HaveOccurred())
Expect(n).To(Equal(int64(1)))
@ -255,6 +255,23 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
})
It("enqueues only the oldest stale absent states up to the limit", func() {
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
now := time.Now()
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "oldest", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-72 * time.Hour)})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "older", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-60 * time.Hour)})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "old", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-48 * time.Hour)})).To(Succeed())
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, now.Add(-24*time.Hour), 2)
Expect(err).ToNot(HaveOccurred())
Expect(n).To(Equal(int64(2)))
items, err := repo.DequeueBatch(10)
Expect(err).ToNot(HaveOccurred())
ids := slice.Map(items, func(it model.ArtworkQueueItem) string { return it.ItemID })
Expect(ids).To(ConsistOf("oldest", "older"))
})
It("enqueues entities that have no item_artwork row at all", func() {
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: time.Now()})).To(Succeed())