navidrome/core/artwork/housekeeping.go
Deluan Quintão 295886cb9a
feat(artwork): report what a config-fingerprint backfill enqueued (#6010)
* feat(artwork): report what a config-fingerprint backfill enqueued

A backfill re-resolves every entity, and on a large library that is tens of thousands of
external agent calls. It announced itself with a single line carrying nothing but an elapsed
time, so the size of the job was invisible until the request volume showed up hours later.

Log the item count, the per-kind breakdown, and a ceiling on the external lookups the queued
work can cost. The ceiling reuses ExternalLookupsPerItem, the same estimator behind the
`artwork reprocess` preview, so the two agree on what an item can cost.

backfill now returns a summary instead of a bare bool, which keeps the counts assertable
without capturing log output. Worker.Backfill keeps its (bool, error) signature, so its caller
is unchanged, and it reads the agent count off its own resolver.

* refactor(artwork): share the image-agent count and take it lazily

Counting image agents was written twice, once in the CLI for the `artwork reprocess` preview and
again as a resolver method for the backfill log. Two copies of "which agents count as image
agents" can drift, and the CLI estimate and the server log would then disagree silently.

Move the derivation next to the type it builds, as NewImageAgentCount, and call it from both.
The resolver method goes away with it: hanging the census on the resolver forced two nil guards
that its only caller could never trigger, because Worker always builds a resolver with agents.
The Worker keeps the *agents.Agents it is already handed instead of reaching through the
processor and resolver to find it.

Pass the count as a func. Building the agent list constructs every enabled agent (each one an
HTTP client and a cache goroutine) only to take its length, and a backfill returns early on an
unchanged fingerprint, which is what happens on nearly every restart.

* docs(artwork): say what the backfill lookup estimate does not bound

The comment called the number a ceiling, which the CLI comment on the same estimate already
contradicts: externalEstimate "claims no bound". Both are right about the local-source case and
only one of them mentions that a retried item asks its agents again.
2026-08-21 20:27:42 -04:00

179 lines
6.9 KiB
Go

package artwork
import (
"context"
"fmt"
"slices"
"strconv"
"strings"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/core/auth"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
"github.com/zeebo/xxh3"
)
// StaleAbsentAge is how long an absent state is trusted before a recheck retries it.
const StaleAbsentAge = 7 * 24 * time.Hour
// StaleAbsentRecheckBatch caps how many absent states each hourly tick re-queues per kind,
// oldest first, so external agents see a flat drip instead of a daily burst.
const StaleAbsentRecheckBatch = 100
// RecheckKinds omits media files: they resolve embedded-only, at scan or on view.
var RecheckKinds = []model.Kind{
model.KindArtistArtwork, model.KindAlbumArtwork, model.KindPlaylistArtwork, model.KindRadioArtwork,
}
// KeepsState reports whether a kind is recorded in item_artwork and the artwork queue. Disc
// artwork is read through on every request and cached by content key, so it has neither.
func KeepsState(kind model.Kind) bool { return kind != model.KindDiscArtwork }
// RefreshableKinds is every kind Refresh can clear and re-queue, so it holds exactly the kinds
// KeepsState admits. Media files are absent from RecheckKinds but belong here: the worker
// resolves them, it just never revisits them on its own.
var RefreshableKinds = append(slices.Clone(RecheckKinds), model.KindMediaFileArtwork)
// hasRecheckPath reports whether a periodic job will revisit this kind, making an absent settle recoverable.
func hasRecheckPath(prefix string) bool {
kind, ok := model.ParseKind(prefix)
return ok && slices.Contains(RecheckKinds, kind)
}
// artworkEpoch invalidates all resolution state when bumped; bump it whenever resolution semantics change.
const artworkEpoch = 1
// FingerprintInput is one config value the fingerprint covers, named after the setting it came from.
type FingerprintInput struct {
Name string
Value string
}
// FingerprintInputs is the single listing of what ConfigFingerprint hashes.
func FingerprintInputs() []FingerprintInput {
return []FingerprintInput{
{"CoverArtPriority", conf.Server.CoverArtPriority},
{"ArtistArtPriority", conf.Server.ArtistArtPriority},
{"ArtistImageFolder", conf.Server.ArtistImageFolder},
{"Agents", conf.Server.Agents},
{"EnableExternalServices", strconv.FormatBool(conf.Server.EnableExternalServices)},
{"EnableM3UExternalAlbumArt", strconv.FormatBool(conf.Server.EnableM3UExternalAlbumArt)},
}
}
// ConfigFingerprint covers the inputs that affect resolution outcomes; a change invalidates stored state.
func ConfigFingerprint() string {
values := slice.Map(FingerprintInputs(), func(i FingerprintInput) string { return i.Value })
raw := fmt.Sprintf("%s|%d", strings.Join(values, "|"), artworkEpoch)
return fmt.Sprintf("%016x", xxh3.Hash([]byte(raw)))
}
// backfillSummary is what a backfill enqueued. MaxExternalLookups is an upper estimate for one
// attempt per item, not a bound: a local hit ends the walk, and a retry asks the agents again.
type backfillSummary struct {
Ran bool
PerKind map[string]int64
Items int64
MaxExternalLookups int64
}
// backfill enqueues artwork resolution for every entity when the config fingerprint changed.
func backfill(ctx context.Context, ds model.DataStore, agentCount func() ImageAgentCount) (backfillSummary, error) {
start := time.Now()
ctx = auth.WithAdminUser(ctx, ds)
current := ConfigFingerprint()
props := ds.Property(ctx)
stored, err := props.DefaultGet(consts.ArtConfFingerprintPropertyKey, "")
if err != nil {
return backfillSummary{}, err
}
if stored == current {
return backfillSummary{}, nil
}
// Artists first: few entities, most external-dependent, so they get a queue headstart.
kinds := []struct {
kind model.Kind
fetch func() ([]string, error)
}{
{model.KindArtistArtwork, func() ([]string, error) { return ds.Artist(ctx).GetAllIDs() }},
{model.KindAlbumArtwork, func() ([]string, error) { return ds.Album(ctx).GetAllIDs() }},
{model.KindPlaylistArtwork, func() ([]string, error) { return ds.Playlist(ctx).GetAllIDs() }},
{model.KindRadioArtwork, func() ([]string, error) { return ds.Radio(ctx).GetAllIDs() }},
}
// Counted here, not by the caller: building the agent list constructs every enabled agent, and
// an unchanged fingerprint returns above without ever needing the number.
agents := agentCount()
summary := backfillSummary{Ran: true, PerKind: map[string]int64{}}
for _, k := range kinds {
ids, err := k.fetch()
if err != nil {
return backfillSummary{}, err
}
if err := enqueueBackfillKind(ctx, ds, k.kind, ids); err != nil {
return backfillSummary{}, err
}
n := int64(len(ids))
summary.PerKind[k.kind.Prefix()] = n
summary.Items += n
summary.MaxExternalLookups += n * ExternalLookupsPerItem(k.kind, agents)
}
if err := props.Put(consts.ArtConfFingerprintPropertyKey, current); err != nil {
return backfillSummary{}, err
}
log.Info(ctx, "Artwork: Config fingerprint changed, backfill enqueued", "items", summary.Items,
"byKind", summary.PerKind, "maxExternalLookups", summary.MaxExternalLookups,
"elapsed", time.Since(start))
return summary, nil
}
func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind model.Kind, ids []string) error {
if len(ids) == 0 {
return nil
}
items := slice.Map(ids, func(id string) model.ArtworkQueueItem {
return model.ArtworkQueueItem{
ItemKind: kind.Prefix(), ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill,
}
})
return ds.ArtworkQueue(ctx).Enqueue(items...)
}
func enqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {
cutoff := time.Now().Add(-StaleAbsentAge)
queue := ds.ArtworkQueue(ctx)
for _, kind := range RecheckKinds {
if _, err := queue.EnqueueStaleAbsent(kind, cutoff, StaleAbsentRecheckBatch); err != nil {
return err
}
}
return nil
}
// enqueueMissingAll is the safety net for entities a scan never enqueued (added between scans, or scanner off).
func enqueueMissingAll(ctx context.Context, ds model.DataStore) error {
queue := ds.ArtworkQueue(ctx)
for _, kind := range RecheckKinds {
if _, err := queue.EnqueueAllMissing(kind, model.ArtworkPriorityRecheck); err != nil {
return err
}
}
return nil
}
// Refresh drops an item's resolved artwork state and re-queues it at Bump priority.
func Refresh(ctx context.Context, ds model.DataStore, kind model.Kind, id string) error {
if err := ds.Artwork(ctx).DeleteForItems(kind, []string{id}); err != nil {
return fmt.Errorf("clearing artwork state: %w", err)
}
item := model.ArtworkQueueItem{ItemKind: kind.Prefix(), ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBump}
if err := ds.ArtworkQueue(ctx).Enqueue(item); err != nil {
return fmt.Errorf("enqueuing artwork refresh: %w", err)
}
return nil
}