diff --git a/cmd/root.go b/cmd/root.go index 9e2b38cd8..4df5c1cb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -88,6 +89,9 @@ func runNavidrome(ctx context.Context) { g.Go(startInsightsCollector(ctx)) g.Go(scheduleDBAnalyzer(ctx)) g.Go(startPluginManager(ctx)) + artworkWorker := CreateArtworkWorker() + g.Go(startArtworkWorker(ctx, artworkWorker)) + g.Go(scheduleArtworkHousekeeping(ctx, artworkWorker)) g.Go(runInitialScan(ctx)) if conf.Server.Scanner.Enabled { g.Go(startScanWatcher(ctx)) @@ -344,6 +348,60 @@ func startPlaybackServer(ctx context.Context) func() error { } } +// startArtworkWorker starts the background artwork acquisition worker. It always +// runs; the queue is simply empty until something enqueues work into it. +func startArtworkWorker(ctx context.Context, worker *artwork.Worker) func() error { + return func() error { + log.Info(ctx, "Starting artwork worker") + return worker.Run(ctx) + } +} + +// scheduleArtworkHousekeeping runs the startup fingerprint backfill and registers the +// recurring stale-absent recheck and prune jobs. Scan-triggered prune lands in a later phase. +func scheduleArtworkHousekeeping(ctx context.Context, worker *artwork.Worker) func() error { + return func() error { + ds := CreateDataStore() + schedulerInstance := scheduler.GetInstance() + + if _, err := schedulerInstance.Add(consts.ArtworkStaleAbsentRecheckSchedule, func() { + if err := artwork.EnqueueStaleAbsentAll(ctx, ds); err != nil { + log.Error(ctx, "Error enqueueing stale artwork rechecks", err) + } + }); err != nil { + log.Error(ctx, "Error scheduling artwork stale-absent recheck", err) + } + + if _, err := schedulerInstance.Add(consts.ArtworkPruneSchedule, func() { + if err := worker.RunPrune(ctx); err != nil { + log.Error(ctx, "Error running artwork prune", err) + } + }); err != nil { + log.Error(ctx, "Error scheduling artwork prune", err) + } + + backfilled, err := artwork.Backfill(ctx, ds) + if err != nil { + log.Error(ctx, "Error running artwork backfill", err) + return nil + } + if !backfilled { + return nil + } + log.Info(ctx, "Artwork backfill enqueued, scheduling a follow-up prune") + timer := time.NewTimer(consts.ArtworkPostBackfillPruneDelay) + defer timer.Stop() + select { + case <-timer.C: + if err := worker.RunPrune(ctx); err != nil { + log.Error(ctx, "Error running post-backfill artwork prune", err) + } + case <-ctx.Done(): + } + return nil + } +} + // startPluginManager starts the plugin manager, if configured. func startPluginManager(ctx context.Context) func() error { return func() error { diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index c4a797ef4..44c4cb46d 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -236,6 +236,21 @@ func GetPlaybackServer() playback.PlaybackServer { return playbackServer } +func CreateArtworkWorker() *artwork.Worker { + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) + imageStore := artwork.ProvideImageStore() + broker := events.GetBroker() + metricsMetrics := metrics.GetPrometheusInstance(dataStore) + manager := plugins.GetManager(dataStore, broker, metricsMetrics) + agentsAgents := agents.GetAgents(dataStore, manager) + matcherMatcher := matcher.New(dataStore) + provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher) + fFmpeg := ffmpeg.New() + worker := artwork.NewWorker(dataStore, imageStore, provider, fFmpeg) + return worker +} + func getPluginManager() *plugins.Manager { sqlDB := db.Db() dataStore := persistence.New(sqlDB) diff --git a/cmd/wire_injectors.go b/cmd/wire_injectors.go index 94faa5af3..a835e5c4c 100644 --- a/cmd/wire_injectors.go +++ b/cmd/wire_injectors.go @@ -136,6 +136,12 @@ func GetPlaybackServer() playback.PlaybackServer { )) } +func CreateArtworkWorker() *artwork.Worker { + panic(wire.Build( + allProviders, + )) +} + func getPluginManager() *plugins.Manager { panic(wire.Build( allProviders, diff --git a/conf/configuration.go b/conf/configuration.go index 83793bd43..9eeda89de 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -57,6 +57,8 @@ type configOptions struct { ImageCacheSize string AlbumPlayCountMode string EnableArtworkPrecache bool + ArtworkWorkerConcurrency int + ArtworkExternalMaxRPS int AutoImportPlaylists bool DefaultPlaylistPublicVisibility bool PlaylistsPath string @@ -346,6 +348,8 @@ func Load(noConfigDump bool) { mapDeprecatedOption("CoverJpegQuality", "CoverArtQuality") mapDeprecatedOption("SimilarSongsMatchThreshold", "Matcher.FuzzyThreshold") mapDeprecatedOption("EnableTranscodingCancellation", "Transcoding.EnableCancellation") + mapDeprecatedOption("DevArtworkWorkerConcurrency", "ArtworkWorkerConcurrency") + mapDeprecatedOption("DevArtworkExternalRPS", "ArtworkExternalMaxRPS") err := viper.Unmarshal(&Server, viper.DecodeHook( mapstructure.ComposeDecodeHookFunc( @@ -900,6 +904,8 @@ func setViperDefaults() { viper.SetDefault("devartworkthrottlebackloglimit", consts.RequestThrottleBacklogLimit) viper.SetDefault("devartworkthrottlebacklogtimeout", consts.RequestThrottleBacklogTimeout) viper.SetDefault("devartworkthrottlebuffered", true) + viper.SetDefault("artworkworkerconcurrency", 4) + viper.SetDefault("artworkexternalmaxrps", 2) viper.SetDefault("devartistinfotimetolive", consts.ArtistInfoTimeToLive) viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive) viper.SetDefault("devexternalscanner", true) diff --git a/consts/consts.go b/consts/consts.go index f453ac125..0a244f55f 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -35,6 +35,10 @@ const ( DBAnalyzeCheckSchedule = "@every 30m" DBAnalyzeMaxAge = 24 * time.Hour + ArtworkStaleAbsentRecheckSchedule = "@every 1h" + ArtworkPruneSchedule = "@daily" + ArtworkPostBackfillPruneDelay = 10 * time.Minute + // DefaultEncryptionKey This is the encryption key used if none is specified in the `PasswordEncryptionKey` option // Never ever change this! Or it will break all Navidrome installations that don't set the config option DefaultEncryptionKey = "just for obfuscation" diff --git a/core/artwork/artwork_suite_test.go b/core/artwork/artwork_suite_test.go index d42d7f3e4..e2a1a591c 100644 --- a/core/artwork/artwork_suite_test.go +++ b/core/artwork/artwork_suite_test.go @@ -15,9 +15,24 @@ import ( "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/goleak" ) func TestArtwork(t *testing.T) { + // Runs unconditionally: the two leaks below are pre-existing and out of this + // package's control, so they're ignored by exact top-function instead. + defer goleak.VerifyNone(t, + goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"), + // notify's own init() starts a singleton tree the moment it's imported (via + // core/storage/local or plugins); recursive on darwin, nonrecursive on linux. + goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*recursiveTree).dispatch"), + goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*nonrecursiveTree).dispatch"), + goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*nonrecursiveTree).internal"), + // The old cache_warmer.go starts a goroutine per NewCacheWarmer call with + // no shutdown path (dark-launch target for Phase 2, not touched here). + goleak.IgnoreTopFunction("github.com/navidrome/navidrome/core/artwork.(*cacheWarmer).waitSignal"), + ) + tests.Init(t, false) log.SetLevel(log.LevelFatal) RegisterFailHandler(Fail) diff --git a/core/artwork/blurhash/blurhash.go b/core/artwork/blurhash/blurhash.go new file mode 100644 index 000000000..302ec83c0 --- /dev/null +++ b/core/artwork/blurhash/blurhash.go @@ -0,0 +1,177 @@ +// Package blurhash implements the blurhash encoding algorithm (https://github.com/woltapp/blurhash), +// matching Jellyfin's parameters so clients tuned against Jellyfin see equivalent hashes. +package blurhash + +import ( + "errors" + "image" + "image/draw" + "math" + "strings" + "sync" + + xdraw "golang.org/x/image/draw" +) + +const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~" + +// maxInputSize matches Jellyfin: larger inputs are slower with no visually discernible difference. +const maxInputSize = 128 + +// Components picks x/y component counts for an image, targeting ~16 near-square tiles (Jellyfin's formula). +func Components(width, height int) (int, int) { + if width <= 0 || height <= 0 { + return 0, 0 + } + xf := math.Sqrt(16.0 * float64(width) / float64(height)) + yf := xf * float64(height) / float64(width) + return min(int(xf)+1, 9), min(int(yf)+1, 9) +} + +// Encode returns the blurhash of img using xComp x yComp components. +func Encode(img image.Image, xComp, yComp int) (string, error) { + if xComp < 1 || xComp > 9 || yComp < 1 || yComp > 9 { + return "", errors.New("blurhash: components must be between 1 and 9") + } + rgba := toRGBA(downscale(img)) + bounds := rgba.Bounds() + w, h := bounds.Dx(), bounds.Dy() + if w == 0 || h == 0 { + return "", errors.New("blurhash: empty image") + } + + cosX := make([][]float64, xComp) + for i := range cosX { + cosX[i] = make([]float64, w) + for x := range cosX[i] { + cosX[i][x] = math.Cos(math.Pi * float64(i) * float64(x) / float64(w)) + } + } + cosY := make([][]float64, yComp) + for j := range cosY { + cosY[j] = make([]float64, h) + for y := range cosY[j] { + cosY[j][y] = math.Cos(math.Pi * float64(j) * float64(y) / float64(h)) + } + } + + lin := srgbToLinearTable() + factors := make([][3]float64, xComp*yComp) + for y := 0; y < h; y++ { + row := rgba.Pix[y*rgba.Stride:] + for x := 0; x < w; x++ { + p := x * 4 + lr, lg, lb := lin[row[p]], lin[row[p+1]], lin[row[p+2]] + for j := 0; j < yComp; j++ { + for i := 0; i < xComp; i++ { + basis := cosX[i][x] * cosY[j][y] + f := &factors[j*xComp+i] + f[0] += basis * lr + f[1] += basis * lg + f[2] += basis * lb + } + } + } + } + for idx := range factors { + norm := 2.0 + if idx == 0 { + norm = 1.0 + } + scale := norm / float64(w*h) + factors[idx][0] *= scale + factors[idx][1] *= scale + factors[idx][2] *= scale + } + + var sb strings.Builder + sb.WriteString(Encode83((xComp-1)+(yComp-1)*9, 1)) + + ac := factors[1:] + maxVal := 1.0 + if len(ac) > 0 { + actualMax := 0.0 + for _, f := range ac { + actualMax = max(actualMax, math.Abs(f[0]), math.Abs(f[1]), math.Abs(f[2])) + } + quantMax := int(math.Max(0, math.Min(82, math.Floor(actualMax*166-0.5)))) + maxVal = float64(quantMax+1) / 166 + sb.WriteString(Encode83(quantMax, 1)) + } else { + sb.WriteString(Encode83(0, 1)) + } + + dc := factors[0] + sb.WriteString(Encode83(linearToSRGB(dc[0])<<16|linearToSRGB(dc[1])<<8|linearToSRGB(dc[2]), 4)) + for _, f := range ac { + sb.WriteString(Encode83(quantAC(f[0], maxVal)*19*19+quantAC(f[1], maxVal)*19+quantAC(f[2], maxVal), 2)) + } + return sb.String(), nil +} + +// toRGBA gives the pixel loop direct Pix access, avoiding a per-pixel allocation through the +// image.At interface (~16k allocs per encode). +func toRGBA(img image.Image) *image.RGBA { + if rgba, ok := img.(*image.RGBA); ok { + return rgba + } + b := img.Bounds() + dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) + draw.Draw(dst, dst.Bounds(), img, b.Min, draw.Src) + return dst +} + +var srgbToLinearTable = sync.OnceValue(func() *[256]float64 { + var t [256]float64 + for i := range t { + t[i] = srgbToLinear(i) + } + return &t +}) + +func downscale(img image.Image) image.Image { + b := img.Bounds() + w, h := b.Dx(), b.Dy() + if w <= maxInputSize && h <= maxInputSize { + return img + } + scale := float64(maxInputSize) / float64(max(w, h)) + dst := image.NewRGBA(image.Rect(0, 0, max(1, int(float64(w)*scale)), max(1, int(float64(h)*scale)))) + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), img, b, draw.Src, nil) + return dst +} + +func quantAC(v, maxVal float64) int { + return int(math.Max(0, math.Min(18, math.Floor(signPow(v/maxVal, 0.5)*9+9.5)))) +} + +func signPow(v, exp float64) float64 { + return math.Copysign(math.Pow(math.Abs(v), exp), v) +} + +func srgbToLinear(v int) float64 { + f := float64(v) / 255 + if f <= 0.04045 { + return f / 12.92 + } + return math.Pow((f+0.055)/1.055, 2.4) +} + +func linearToSRGB(v float64) int { + v = math.Min(math.Max(0, v), 1) + if v <= 0.0031308 { + return int(v*12.92*255 + 0.5) + } + return int((1.055*math.Pow(v, 1/2.4)-0.055)*255 + 0.5) +} + +// Encode83 encodes value as a fixed-width, big-endian base83 string of the given length, using the +// blurhash spec's alphabet. +func Encode83(value, length int) string { + b := make([]byte, length) + for i := length - 1; i >= 0; i-- { + b[i] = alphabet[value%83] + value /= 83 + } + return string(b) +} diff --git a/core/artwork/blurhash/blurhash_bench_test.go b/core/artwork/blurhash/blurhash_bench_test.go new file mode 100644 index 000000000..43b593eb5 --- /dev/null +++ b/core/artwork/blurhash/blurhash_bench_test.go @@ -0,0 +1,41 @@ +package blurhash_test + +import ( + "fmt" + "image" + "image/color" + "testing" + + "github.com/navidrome/navidrome/core/artwork/blurhash" +) + +// benchImage builds a deterministic gradient so runs are comparable across revisions. +func benchImage(size int) image.Image { + img := image.NewNRGBA(image.Rect(0, 0, size, size)) + for y := 0; y < size; y++ { + for x := 0; x < size; x++ { + img.SetNRGBA(x, y, color.NRGBA{ + R: uint8(255 * x / size), + G: uint8(255 * y / size), + B: uint8((x + y) * 255 / (2 * size)), + A: 255, + }) + } + } + return img +} + +func BenchmarkEncode(b *testing.B) { + for _, size := range []int{100, 300, 600, 900, 1200, 1500} { + img := benchImage(size) + x, y := blurhash.Components(size, size) + b.Run(fmt.Sprintf("%dx%d", size, size), func(b *testing.B) { + b.ReportAllocs() + for range b.N { + if _, err := blurhash.Encode(img, x, y); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/core/artwork/blurhash/blurhash_suite_test.go b/core/artwork/blurhash/blurhash_suite_test.go new file mode 100644 index 000000000..b0e96d7a7 --- /dev/null +++ b/core/artwork/blurhash/blurhash_suite_test.go @@ -0,0 +1,17 @@ +package blurhash_test + +import ( + "testing" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBlurHash(t *testing.T) { + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + RegisterFailHandler(Fail) + RunSpecs(t, "BlurHash Suite") +} diff --git a/core/artwork/blurhash/blurhash_test.go b/core/artwork/blurhash/blurhash_test.go new file mode 100644 index 000000000..18dd86b99 --- /dev/null +++ b/core/artwork/blurhash/blurhash_test.go @@ -0,0 +1,113 @@ +package blurhash_test + +import ( + "image" + "image/color" + "strings" + + "github.com/navidrome/navidrome/core/artwork/blurhash" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~" + +func decode83(s string) int { + v := 0 + for _, c := range s { + v = v*83 + strings.IndexRune(alphabet, c) + } + return v +} + +func solidImage(w, h int, c color.NRGBA) image.Image { + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.SetNRGBA(x, y, c) + } + } + return img +} + +func gradientImage(w, h int) image.Image { + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(255 * x / w), G: uint8(255 * y / h), B: 128, A: 255}) + } + } + return img +} + +var _ = Describe("Components", func() { + DescribeTable("derives component counts from aspect ratio (Jellyfin formula)", + func(w, h, expectedX, expectedY int) { + x, y := blurhash.Components(w, h) + Expect(x).To(Equal(expectedX)) + Expect(y).To(Equal(expectedY)) + }, + Entry("square album art", 600, 600, 5, 5), + Entry("small square", 1, 1, 5, 5), + Entry("landscape 16:9", 1920, 1080, 6, 4), + Entry("portrait 9:16", 1080, 1920, 4, 6), + Entry("extreme landscape capped at 9", 10000, 100, 9, 1), + Entry("zero width", 0, 600, 0, 0), + Entry("zero height", 600, 0, 0, 0), + ) +}) + +var _ = Describe("Encode", func() { + It("rejects out-of-range components", func() { + _, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 0, 5) + Expect(err).To(HaveOccurred()) + _, err = blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 5, 10) + Expect(err).To(HaveOccurred()) + }) + + It("produces the spec-mandated length", func() { + // 1 (size flag) + 1 (max AC) + 4 (DC) + 2 per AC component + h, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{R: 10, G: 20, B: 30, A: 255}), 4, 3) + Expect(err).ToNot(HaveOccurred()) + Expect(h).To(HaveLen(4 + 2 + 2*(4*3-1))) + }) + + It("encodes the size flag as the first character", func() { + h, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 4, 3) + Expect(err).ToNot(HaveOccurred()) + Expect(decode83(h[:1])).To(Equal((4 - 1) + (3-1)*9)) + }) + + It("stores the average color in the DC component", func() { + h, err := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 200, G: 100, B: 50, A: 255}), 4, 3) + Expect(err).ToNot(HaveOccurred()) + dc := decode83(h[2:6]) + Expect(dc >> 16).To(BeNumerically("~", 200, 1)) + Expect((dc >> 8) & 0xFF).To(BeNumerically("~", 100, 1)) + Expect(dc & 0xFF).To(BeNumerically("~", 50, 1)) + }) + + It("is deterministic", func() { + img := gradientImage(64, 64) + h1, err1 := blurhash.Encode(img, 5, 5) + h2, err2 := blurhash.Encode(img, 5, 5) + Expect(err1).ToNot(HaveOccurred()) + Expect(err2).ToNot(HaveOccurred()) + Expect(h1).To(Equal(h2)) + }) + + It("produces different hashes for different images", func() { + h1, _ := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 255, A: 255}), 4, 4) + h2, _ := blurhash.Encode(gradientImage(16, 16), 4, 4) + Expect(h1).ToNot(Equal(h2)) + }) + + It("downscales large images internally without changing the result materially", func() { + // A 1000px solid image must encode fine and carry the same DC as its small version. + big, err := blurhash.Encode(solidImage(1000, 1000, color.NRGBA{R: 60, G: 120, B: 180, A: 255}), 5, 5) + Expect(err).ToNot(HaveOccurred()) + small, err := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 60, G: 120, B: 180, A: 255}), 5, 5) + Expect(err).ToNot(HaveOccurred()) + Expect(big[2:6]).To(Equal(small[2:6])) + }) +}) diff --git a/core/artwork/e2e/helpers_test.go b/core/artwork/e2e/helpers_test.go index e3abca097..4e99f7a5c 100644 --- a/core/artwork/e2e/helpers_test.go +++ b/core/artwork/e2e/helpers_test.go @@ -177,6 +177,9 @@ func (n *noopProvider) TopSongs(context.Context, string, int) (model.MediaFiles, func (n *noopProvider) ArtistImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } +func (n *noopProvider) ArtistImageResult(context.Context, string) (*url.URL, error) { + return nil, model.ErrNotFound +} func (n *noopProvider) AlbumImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go new file mode 100644 index 000000000..f45c62ab6 --- /dev/null +++ b/core/artwork/housekeeping.go @@ -0,0 +1,102 @@ +package artwork + +import ( + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/auth" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against +// to detect artwork-affecting config changes across restarts. +const FingerprintPropertyKey = "artwork.fingerprint" + +// staleAbsentAge is how old an absent resolution must be before the recheck job retries it. +const staleAbsentAge = 24 * time.Hour + +// staleAbsentKinds are the item kinds eligible for the periodic stale-absent recheck. +var staleAbsentKinds = []string{"ar", "al", "pl", "ra"} + +// Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a +// change means previously resolved (or absent) state may no longer be correct. +func Fingerprint() string { + raw := fmt.Sprintf("%s|%s|%s|%s|%t|%t|%s", + conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, conf.Server.ArtistImageFolder, + conf.Server.Agents, conf.Server.EnableExternalServices, conf.Server.EnableM3UExternalAlbumArt, consts.Version) + sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive + return hex.EncodeToString(sum[:]) +} + +// Backfill enqueues artwork resolution for every entity when the config fingerprint changed +// (or was never stored), artists first so those pages resolve before the larger backlog. +func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { + ctx = auth.WithAdminUser(ctx, ds) + current := Fingerprint() + props := ds.Property(ctx) + stored, err := props.DefaultGet(FingerprintPropertyKey, "") + if err != nil { + return false, err + } + if stored == current { + return false, nil + } + + // Artists first: few entities, most external-dependent, so they get queue headstart. + kinds := []struct { + kind string + fetch func() ([]string, error) + }{ + {"ar", func() ([]string, error) { return ds.Artist(ctx).GetAllIDs() }}, + {"al", func() ([]string, error) { return ds.Album(ctx).GetAllIDs() }}, + {"pl", func() ([]string, error) { return ds.Playlist(ctx).GetAllIDs() }}, + {"ra", func() ([]string, error) { return ds.Radio(ctx).GetAllIDs() }}, + } + for _, k := range kinds { + ids, err := k.fetch() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, k.kind, ids); err != nil { + return false, err + } + } + + if err := props.Put(FingerprintPropertyKey, current); err != nil { + return false, err + } + log.Info(ctx, "Artwork: config fingerprint changed, backfill enqueued") + return true, nil +} + +func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, ids []string) error { + if len(ids) == 0 { + return nil + } + items := make([]model.ArtworkQueueItem, len(ids)) + for i, id := range ids { + items[i] = model.ArtworkQueueItem{ + ItemKind: kind, ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill, + } + } + return ds.ArtworkQueue(ctx).Enqueue(items...) +} + +// EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across +// every artwork-bearing kind, for the periodic recheck job. +func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error { + cutoff := time.Now().Add(-staleAbsentAge) + queue := ds.ArtworkQueue(ctx) + for _, kind := range staleAbsentKinds { + if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil { + return err + } + } + return nil +} diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go new file mode 100644 index 000000000..1df8312e8 --- /dev/null +++ b/core/artwork/housekeeping_test.go @@ -0,0 +1,226 @@ +package artwork + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// visibilityPlaylistDS models playlist_repository's userFilter: a private playlist is only +// visible when the ctx carries an admin, so headless work must wrap ctx with one first. +type visibilityPlaylistDS struct { + *tests.MockDataStore + private model.Playlist + tracks model.PlaylistTrackRepository +} + +func (v *visibilityPlaylistDS) Playlist(ctx context.Context) model.PlaylistRepository { + repo := tests.CreateMockPlaylistRepo() + repo.TracksRepo = v.tracks + if u, ok := request.UserFrom(ctx); ok && u.IsAdmin { + repo.SetData(model.Playlists{v.private}) + } + return repo +} + +func adminUserRepo() *tests.MockedUserRepo { + repo := tests.CreateMockUserRepo() + Expect(repo.Put(&model.User{ID: "admin", UserName: "admin", IsAdmin: true})).To(Succeed()) + return repo +} + +// orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can +// assert phase ordering (artists-first) that same-priority timestamps can't guarantee. +type orderTrackingQueueRepo struct { + *tests.MockArtworkQueueRepo + callKinds []string +} + +func (o *orderTrackingQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { + if len(items) > 0 { + o.callKinds = append(o.callKinds, items[0].ItemKind) + } + return o.MockArtworkQueueRepo.Enqueue(items...) +} + +var _ = Describe("Housekeeping", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + queueRepo *orderTrackingQueueRepo + propRepo *tests.MockedPropertyRepo + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + conf.Server.CoverArtPriority = "embedded, folder" + conf.Server.ArtistArtPriority = "artist.jpg" + conf.Server.Agents = "spotify" + conf.Server.EnableExternalServices = true + + queueRepo = &orderTrackingQueueRepo{MockArtworkQueueRepo: tests.CreateMockArtworkQueueRepo()} + propRepo = &tests.MockedPropertyRepo{} + ds = &tests.MockDataStore{MockedArtworkQueue: queueRepo, MockedProperty: propRepo} + }) + + seedEntities := func() { + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar1"}, {ID: "ar2"}}) + ds.MockedArtist = artistRepo + + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{{ID: "al1"}}) + ds.MockedAlbum = albumRepo + + playlistRepo := tests.CreateMockPlaylistRepo() + playlistRepo.SetData(model.Playlists{{ID: "pl1"}}) + ds.MockedPlaylist = playlistRepo + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.All = model.Radios{{ID: "ra1"}} + ds.MockedRadio = radioRepo + } + + Describe("Fingerprint", func() { + It("changes when a fingerprint-affecting config value changes", func() { + f1 := Fingerprint() + conf.Server.CoverArtPriority = "folder, embedded" + f2 := Fingerprint() + Expect(f1).NotTo(Equal(f2)) + }) + + It("changes when ArtistImageFolder changes", func() { + conf.Server.ArtistImageFolder = "/before" + f1 := Fingerprint() + conf.Server.ArtistImageFolder = "/after" + Expect(Fingerprint()).NotTo(Equal(f1)) + }) + + It("changes when EnableM3UExternalAlbumArt is toggled", func() { + conf.Server.EnableM3UExternalAlbumArt = false + f1 := Fingerprint() + conf.Server.EnableM3UExternalAlbumArt = true + Expect(Fingerprint()).NotTo(Equal(f1)) + }) + }) + + Describe("Backfill", func() { + It("enqueues nothing and returns false when the stored fingerprint matches", func() { + seedEntities() + Expect(propRepo.Put(FingerprintPropertyKey, Fingerprint())).To(Succeed()) + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeFalse()) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + }) + + It("runs the backfill when no fingerprint was ever stored", func() { + seedEntities() + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(int64(5))) // 2 artists + 1 album + 1 playlist + 1 radio + + stored, err := propRepo.Get(FingerprintPropertyKey) + Expect(err).ToNot(HaveOccurred()) + Expect(stored).To(Equal(Fingerprint())) + }) + + It("enqueues a private playlist by resolving it under an admin context", func() { + ds.MockedUser = adminUserRepo() + vds := &visibilityPlaylistDS{ + MockDataStore: ds, + private: model.Playlist{ID: "plPrivate", OwnerID: "admin"}, + tracks: &tests.MockPlaylistTrackRepo{}, + } + + did, err := Backfill(ctx, vds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "plPrivate")).ToNot(BeNil()) + }) + + It("enqueues artists before albums/playlists/radios, all at Backfill priority", func() { + seedEntities() + Expect(propRepo.Put(FingerprintPropertyKey, "stale-fingerprint")).To(Succeed()) + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + + Expect(queueRepo.callKinds).ToNot(BeEmpty()) + artistCallIdx := -1 + for i, k := range queueRepo.callKinds { + if k == "ar" { + artistCallIdx = i + break + } + } + Expect(artistCallIdx).To(Equal(0), "artists must be the first Enqueue call") + for i, k := range queueRepo.callKinds { + if k != "ar" { + Expect(i).To(BeNumerically(">", artistCallIdx)) + } + } + + for _, it := range queueRepo.Data { + Expect(it.Priority).To(Equal(model.ArtworkPriorityBackfill)) + Expect(it.ItemKind).To(BeElementOf("ar", "al", "pl", "ra")) + } + }) + }) + + Describe("EnqueueStaleAbsentAll", func() { + var artRepo *tests.MockArtworkRepo + + BeforeEach(func() { + artRepo = tests.CreateMockArtworkRepo() + ds.MockedArtwork = artRepo + queueRepo.ItemArtworkSource = artRepo + }) + + It("enqueues only absent entries older than the recheck window, across all kinds", func() { + old := time.Now().Add(-48 * time.Hour) + recent := time.Now().Add(-time.Hour) + + artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["pl-stale"] = model.ItemArtwork{ItemKind: "pl", ItemID: "pl1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["ra-stale"] = model.ItemArtwork{ItemKind: "ra", ItemID: "ra1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + // Not stale: too recent. + artRepo.ItemData["ar-recent"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar2", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: recent} + // Not absent: has a resolved hash. + artRepo.ItemData["al-resolved"] = model.ItemArtwork{ItemKind: "al", ItemID: "al2", ImageType: model.ImageTypePrimary, Hash: "somehash", AttemptedAt: old} + + err := EnqueueStaleAbsentAll(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + + Expect(queueRepo.Data).To(HaveLen(4)) + for _, it := range queueRepo.Data { + Expect(it.Priority).To(Equal(model.ArtworkPriorityRecheck)) + } + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "pl1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ra", "ra1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil()) + }) + }) +}) diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go new file mode 100644 index 000000000..53662f686 --- /dev/null +++ b/core/artwork/image_store.go @@ -0,0 +1,172 @@ +package artwork + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/zeebo/xxh3" +) + +func HashImage(r io.Reader) (string, error) { + d := xxh3.New() + if _, err := io.Copy(d, r); err != nil { + return "", err + } + return fmt.Sprintf("%016x", d.Sum64()), nil +} + +// ImageStore is the content-addressed store for artwork images that have no +// library file backing them (external downloads, embedded extractions, generated). +type ImageStore struct { + root string +} + +func NewImageStore(rootDir string) *ImageStore { + return &ImageStore{root: rootDir} +} + +// ProvideImageStore roots the store in its own subtree under the data folder, so +// Prune's recursive sweep never reaches the per-entity upload folders next to it. +func ProvideImageStore() *ImageStore { + return NewImageStore(filepath.Join(conf.Server.DataFolder.String(), consts.ArtworkFolder, "store")) +} + +// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into +// content-addressed paths and re-derived on Open, so they must be stable across OSes. +func extForMime(m string) string { + switch m { + case "image/jpeg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + } + return ".img" +} + +// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "", +// and malformed persisted hashes must never reach path sharding (slice panics, separators). +func validHash(hash string) bool { + if len(hash) != 16 { + return false + } + for _, c := range []byte(hash) { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func (s *ImageStore) path(hash, mimeType string) string { + return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType)) +} + +func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { + if !validHash(hash) { + return fmt.Errorf("imagestore: invalid hash %q", hash) + } + dst := s.path(hash, mimeType) + if _, err := os.Stat(dst); err == nil { + // A touched mtime marks the file live so a concurrent prune spares it. + now := time.Now() + if err := os.Chtimes(dst, now, now); err == nil { + return nil + } + // touch failed (file likely pruned concurrently) — fall through and write it + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, r); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dst) +} + +func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) { + if !validHash(hash) { + return nil, fmt.Errorf("imagestore: invalid hash %q", hash) + } + return os.Open(s.path(hash, mimeType)) +} + +// Remove deletes the store file unless it is newer than olderThan, in which case +// an overlapping acquisition may have just touched it and be about to commit its row. +func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error { + if !validHash(hash) { + return fmt.Errorf("imagestore: invalid hash %q", hash) + } + path := s.path(hash, mimeType) + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.ModTime().After(olderThan) { + return nil + } + err = os.Remove(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +// Sweep removes store files not accepted by keep. Files modified after cutoff +// (including temp files) are always kept: their acquisition row may not be committed yet. +func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) { + removed := 0 + err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().After(cutoff) { + return nil + } + name := d.Name() + remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window + if !remove { + ext := filepath.Ext(name) + remove = !keep(strings.TrimSuffix(name, ext), ext) + } + if remove { + // #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks + if err := os.Remove(path); err != nil { + return err + } + removed++ + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return removed, nil + } + return removed, err +} diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go new file mode 100644 index 000000000..6537ca906 --- /dev/null +++ b/core/artwork/image_store_test.go @@ -0,0 +1,204 @@ +package artwork + +import ( + "bytes" + "io" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ImageStore", func() { + var store *ImageStore + var root string + + BeforeEach(func() { + root = GinkgoT().TempDir() + store = NewImageStore(root) + }) + + It("hashes deterministically", func() { + h1, err := HashImage(bytes.NewReader([]byte("some image bytes"))) + Expect(err).ToNot(HaveOccurred()) + h2, _ := HashImage(bytes.NewReader([]byte("some image bytes"))) + Expect(h1).To(Equal(h2)) + Expect(h1).To(HaveLen(16)) + h3, _ := HashImage(bytes.NewReader([]byte("other bytes"))) + Expect(h3).ToNot(Equal(h1)) + }) + + It("writes sharded and reads back", func() { + data := []byte("jpeg-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + got, _ := io.ReadAll(rc) + Expect(got).To(Equal(data)) + }) + + It("is idempotent on duplicate writes and preserves the original content", func() { + data := []byte("dup") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + // A duplicate write only touches mtime; passing different bytes under the same + // hash proves the second reader is never consumed to overwrite the file. + Expect(store.Write(h, "image/png", bytes.NewReader([]byte("not-dup")))).To(Succeed()) + + rc, err := store.Open(h, "image/png") + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + got, err := io.ReadAll(rc) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(data)) + }) + + It("refreshes the mtime on a duplicate write", func() { + data := []byte("touch-me") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + + info, err := os.Stat(store.path(h, "image/png")) + Expect(err).ToNot(HaveOccurred()) + Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute))) + }) + + It("rewrites the bytes when the existing file vanished before the liveness touch", func() { + data := []byte("vanishing") + h, _ := HashImage(bytes.NewReader(data)) + for range 10 { + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(os.Remove(store.path(h, "image/png"))).To(Succeed()) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + + rc, err := store.Open(h, "image/png") + Expect(err).ToNot(HaveOccurred()) + got, _ := io.ReadAll(rc) + rc.Close() + Expect(got).To(Equal(data)) + } + }) + + It("returns fs.ErrNotExist for missing images", func() { + _, err := store.Open("beefbeefbeefbeef", "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("removes without error when already gone", func() { + Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed()) + }) + + It("rejects invalid hashes instead of panicking", func() { + for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} { + Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash"))) + _, err := store.Open(h, "image/jpeg") + Expect(err).To(MatchError(ContainSubstring("invalid hash"))) + Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash"))) + } + }) + + It("spares a file newer than the cutoff, removes an aged one", func() { + fresh := []byte("fresh") + hf, _ := HashImage(bytes.NewReader(fresh)) + Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed()) + + aged := []byte("aged") + ha, _ := HashImage(bytes.NewReader(aged)) + Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed()) + + cutoff := time.Now().Add(-time.Hour) + Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed()) + Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed()) + + rc, err := store.Open(hf, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + _, err = store.Open(ha, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("sweeps unknown files, keeps known ones", func() { + d1 := []byte("keep-me") + h1, _ := HashImage(bytes.NewReader(d1)) + Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed()) + d2 := []byte("orphan") + h2, _ := HashImage(bytes.NewReader(d2)) + Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed()) + + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h2, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h1, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("sweeps a stale mime variant of a known hash, keeps the current one", func() { + data := []byte("same-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + + // The recorded mime is image/jpeg, so the .png variant is obsolete. + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool { + return hash == h && ext == ".jpg" + }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h, "image/png") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("keeps young unknown files inside the grace window", func() { + d := []byte("fresh-orphan") + h, _ := HashImage(bytes.NewReader(d)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(0)) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("removes abandoned temp files past the grace window, keeps fresh ones", func() { + oldTmp := filepath.Join(root, ".old.tmp") + Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(oldTmp, old, old)).To(Succeed()) + + freshTmp := filepath.Join(root, ".fresh.tmp") + Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + Expect(oldTmp).ToNot(BeAnExistingFile()) + Expect(freshTmp).To(BeAnExistingFile()) + }) +}) diff --git a/core/artwork/processor.go b/core/artwork/processor.go new file mode 100644 index 000000000..fefdebf84 --- /dev/null +++ b/core/artwork/processor.go @@ -0,0 +1,246 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + "image/draw" + "io" + "time" + + "github.com/navidrome/navidrome/core/artwork/blurhash" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + xdraw "golang.org/x/image/draw" +) + +// outcome tells the worker what to do with the queue row: found/absent +// delete it, failed reschedules it via MarkFailed. +type outcome int + +const ( + outcomeFound outcome = iota + // outcomeFoundStale: state was written and is served, but a higher-priority external + // step failed, so the row must retry (via MarkFailed) to give that source another chance. + outcomeFoundStale + outcomeAbsent + outcomeFailed +) + +// thumbnailSize is the max dimension fed to blurhash. +const thumbnailSize = 128 + +// maxImageBytes caps a resolved image read: a user-editable ExternalImageURL could +// point at an arbitrarily large endpoint, and 20MB is generous for any real cover. +const maxImageBytes = 20 << 20 + +// maxImagePixels caps declared dimensions: a tiny compressed file can declare a +// huge canvas that image.Decode would expand into gigabytes (decompression bomb). +const maxImagePixels = 64 << 20 + +// workerDeps are the collaborators processItem needs; extGate is set by NewWorker in +// production and nil only in tests, where resolveItem falls back to a plain passthrough. +type workerDeps struct { + ds model.DataStore + store *ImageStore + prov external.Provider + ffmpeg ffmpeg.FFmpeg + extGate extGateFunc +} + +// processItem resolves one queue item end to end: find an image, hash/decode/ +// blurhash it, place its bytes, and persist the resulting state. +func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueItem) outcome { + repo := deps.ds.Artwork(ctx) + + res, err := resolveItem(ctx, deps.ds, deps.prov, deps.ffmpeg, item, deps.extGate) + if err != nil { + log.Warn(ctx, "artwork: could not resolve item", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if res.reader == nil { + if res.extError { + // An external source errored/timed out: never settle on absent, keep serving old state. + return outcomeFailed + } + return writeAbsent(ctx, repo, item) + } + defer res.reader.Close() + + data, err := readCapped(res.reader) + if err != nil { + log.Warn(ctx, "artwork: failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, err) + return outcomeFailed + } + log.Debug(ctx, "artwork: read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "bytes", len(data)) + + hash, err := HashImage(bytes.NewReader(data)) + if err != nil { + log.Warn(ctx, "artwork: failed to hash image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + + art, err := repo.GetImage(hash) + switch { + case err == nil: + // Dedup hit: identical bytes already known, reuse dims/mime/blurhash. + case errors.Is(err, model.ErrNotFound): + art, err = decodeArtwork(ctx, hash, data) + if err != nil { + log.Warn(ctx, "artwork: failed to decode resolved image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + default: + log.Warn(ctx, "artwork: failed to look up image hash", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + art.SizeBytes = int64(len(data)) + + sourcePath, refMtime, err := placeBytes(deps.store, art, res, data) + if err != nil { + log.Warn(ctx, "artwork: failed to write image store", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if err := repo.PutImage(art); err != nil { + log.Warn(ctx, "artwork: failed to persist artwork image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if err := repo.PutItemArtwork(&model.ItemArtwork{ + ItemKind: item.ItemKind, + ItemID: item.ItemID, + ImageType: item.ImageType, + Hash: hash, + Source: res.source, + SourcePath: sourcePath, + RefMtime: refMtime, + AttemptedAt: time.Now(), + }); err != nil { + log.Warn(ctx, "artwork: failed to persist item artwork state", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if res.extError { + return outcomeFoundStale + } + return outcomeFound +} + +// writeAbsent records a known-absent state: every local/external source answered definitively "no". +func writeAbsent(ctx context.Context, repo model.ArtworkRepository, item model.ArtworkQueueItem) outcome { + err := repo.PutItemArtwork(&model.ItemArtwork{ + ItemKind: item.ItemKind, + ItemID: item.ItemID, + ImageType: item.ImageType, + AttemptedAt: time.Now(), + }) + if err != nil { + log.Warn(ctx, "artwork: failed to persist absent state", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + return outcomeAbsent +} + +// readCapped reads r, rejecting anything over maxImageBytes. +func readCapped(r io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, maxImageBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxImageBytes { + return nil, fmt.Errorf("image exceeds size cap %d", maxImageBytes) + } + return data, nil +} + +// decodeCapped rejects declared dimensions over maxImagePixels BEFORE the +// full-decode allocation, then decodes. +func decodeCapped(data []byte) (image.Image, string, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, "", fmt.Errorf("decode image config: %w", err) + } + if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels { + return nil, "", fmt.Errorf("image dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels) + } + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, "", fmt.Errorf("decode image: %w", err) + } + return img, format, nil +} + +// decodeArtwork builds a new Artwork row from raw bytes: dimensions, mime and a +// blurhash computed from a downscaled thumbnail. +func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwork, error) { + img, format, err := decodeCapped(data) + if err != nil { + return nil, err + } + + thumb := makeThumbnail(img, thumbnailSize) + xComp, yComp := blurhash.Components(thumb.Bounds().Dx(), thumb.Bounds().Dy()) + bh, err := blurhash.Encode(thumb, xComp, yComp) + if err != nil { + log.Warn(ctx, "artwork: blurhash encoding failed", "hash", hash, err) + bh = "" + } + + return &model.Artwork{ + Hash: hash, + Mime: mimeForFormat(format), + Width: img.Bounds().Dx(), + Height: img.Bounds().Dy(), + BlurHash: bh, + }, nil +} + +// makeThumbnail downscales img to fit within maxSize on its longest side. +// Images within bounds are returned as-is (no upscaling). +func makeThumbnail(img image.Image, maxSize int) image.Image { + b := img.Bounds() + w, h := b.Dx(), b.Dy() + if w <= maxSize && h <= maxSize { + return toFastScaleType(img) + } + scale := float64(maxSize) / float64(max(w, h)) + dst := image.NewRGBA(image.Rect(0, 0, max(1, int(float64(w)*scale)), max(1, int(float64(h)*scale)))) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), toFastScaleType(img), b, draw.Src, nil) + return dst +} + +// isFileBacked reports whether a resolution's bytes already live in a library/upload +// file, so the acquisition must not duplicate them into the content-addressed store. +func isFileBacked(source string) bool { + return source == "folder" || source == "upload" +} + +// placeBytes reports the item's backing-file provenance (folder/upload: image, embedded: audio, +// external/generated: none) and writes the bytes into the store for the non-file-backed sources. +func placeBytes(store *ImageStore, art *model.Artwork, res resolution, data []byte) (sourcePath string, refMtime int64, err error) { + if isFileBacked(res.source) { + return res.sourcePath, res.refMtime, nil + } + if res.source == "embedded" { + sourcePath, refMtime = res.sourcePath, res.refMtime + } + return sourcePath, refMtime, store.Write(art.Hash, art.Mime, bytes.NewReader(data)) +} + +// mimeForFormat maps an image.Decode format name to its MIME type; extForMime +// in image_store.go performs the inverse for content-addressed file paths. +func mimeForFormat(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + case "gif": + return "image/gif" + case "webp": + return "image/webp" + } + return "application/octet-stream" +} diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go new file mode 100644 index 000000000..9ea4338fe --- /dev/null +++ b/core/artwork/processor_test.go @@ -0,0 +1,325 @@ +package artwork + +import ( + "context" + "encoding/binary" + "errors" + "hash/crc32" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// pngHeaderWithDims builds just a PNG signature + IHDR chunk declaring w×h. DecodeConfig +// reads the header without touching pixel data, so the body can be omitted entirely. +func pngHeaderWithDims(w, h uint32) []byte { + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:], w) + binary.BigEndian.PutUint32(ihdr[4:], h) + ihdr[8] = 8 // bit depth + ihdr[9] = 2 // color type: truecolor + chunk := append([]byte("IHDR"), ihdr...) + out := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a} + out = binary.BigEndian.AppendUint32(out, uint32(len(ihdr))) + out = append(out, chunk...) + return binary.BigEndian.AppendUint32(out, crc32.ChecksumIEEE(chunk)) +} + +var _ = Describe("processItem", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + folderRepo *fakeFolderRepo + libRepo *tests.MockLibraryRepo + ffm *tests.MockFFmpeg + prov *fakeExternalProvider + store *ImageStore + artRepo *tests.MockArtworkRepo + repoRoot string + deps *workerDeps + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + folderRepo = &fakeFolderRepo{} + libRepo = &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + ffm = tests.NewMockFFmpeg("") + prov = &fakeExternalProvider{} + artRepo = tests.CreateMockArtworkRepo() + ds = &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + } + ds.MockedAlbum = tests.CreateMockAlbumRepo() + store = NewImageStore(GinkgoT().TempDir()) + deps = &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm} + + conf.Server.CoverArtPriority = "cover.jpg, embedded" + }) + + It("found-folder: persists state from a folder image, writes no store file, keeps sourcePath/refMtime", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}, + }) + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"}) + Expect(out).To(Equal(outcomeFound)) + + ia, err := artRepo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).ToNot(BeEmpty()) + Expect(ia.Source).To(Equal("folder")) + Expect(filepath.ToSlash(ia.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg")) + Expect(ia.RefMtime).To(BeNumerically(">", 0)) + + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred()) + _, err = store.Open(ia.Hash, art.Mime) + Expect(os.IsNotExist(err)).To(BeTrue(), "folder-backed art must not be duplicated into the store") + }) + + It("found-embedded: writes a store file and computes a non-empty blurhash from a real fixture", func() { + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + folderRepo.result = nil + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"}) + Expect(out).To(Equal(outcomeFound)) + + ia, err := artRepo.GetItemArtwork("al", "al2", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("embedded")) + Expect(filepath.ToSlash(ia.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3")) + + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(art.BlurHash).ToNot(BeEmpty()) + + rc, err := store.Open(ia.Hash, art.Mime) + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("absent: no local source and no external error persists a known-absent state", func() { + folderRepo.result = nil + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al3", Name: "Album"}, + }) + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"}) + Expect(out).To(Equal(outcomeAbsent)) + + ia, err := artRepo.GetItemArtwork("al", "al3", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).To(BeEmpty()) + Expect(ia.Source).To(BeEmpty()) + Expect(ia.AttemptedAt).To(BeTemporally("~", time.Now(), time.Second)) + }) + + It("failed-on-extError: leaves the item's state untouched", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al4", Name: "Album"}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork("al", "al4", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("found-stale: a fallback hit after a transient external failure persists state and returns outcomeFoundStale", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "alstale", Name: "Album", FolderIDs: []string{"f1"}}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"}) + Expect(out).To(Equal(outcomeFoundStale)) + + ia, err := artRepo.GetItemArtwork("al", "alstale", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).ToNot(BeEmpty()) + Expect(ia.Source).To(Equal("folder")) + }) + + It("dedup: a second item with identical bytes skips decode and reuses the artwork row", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al5", Name: "Album A", FolderIDs: []string{"f1"}}, + {ID: "al6", Name: "Album B", FolderIDs: []string{"f1"}}, + }) + + out1 := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"}) + Expect(out1).To(Equal(outcomeFound)) + ia1, err := artRepo.GetItemArtwork("al", "al5", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + + // Poison the stored blurhash: if the second item re-decodes instead of + // deduping on hash, this sentinel gets overwritten by a real computed value. + poisoned := artRepo.Data[ia1.Hash] + poisoned.BlurHash = "SENTINEL" + artRepo.Data[ia1.Hash] = poisoned + + out2 := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"}) + Expect(out2).To(Equal(outcomeFound)) + ia2, err := artRepo.GetItemArtwork("al", "al6", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia2.Hash).To(Equal(ia1.Hash)) + + reused, err := artRepo.GetImage(ia1.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(reused.BlurHash).To(Equal("SENTINEL")) + }) + + It("two items, two files, identical bytes: each item keeps its own provenance; the shared artwork row is written once", func() { + // Two distinct library files with byte-identical content resolve to the same + // hash. Provenance is per-item, so neither file's path may overwrite the other. + libRoot := GinkgoT().TempDir() + imgBytes, err := os.ReadFile(filepath.Join(repoRoot, "tests/fixtures/artist/an-album/cover.jpg")) + Expect(err).ToNot(HaveOccurred()) + for sub, mtime := range map[string]int64{"album-a": 1000, "album-b": 2000} { + dir := filepath.Join(libRoot, sub) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + img := filepath.Join(dir, "cover.jpg") + Expect(os.WriteFile(img, imgBytes, 0600)).To(Succeed()) + Expect(os.Chtimes(img, time.Unix(mtime, 0), time.Unix(mtime, 0))).To(Succeed()) + } + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "alA", Name: "Album A", FolderIDs: []string{"fa"}}, + {ID: "alB", Name: "Album B", FolderIDs: []string{"fb"}}, + }) + + folderRepo.result = []model.Folder{{Path: "album-a", ImageFiles: []string{"cover.jpg"}}} + Expect(processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alA"})).To(Equal(outcomeFound)) + iaA, err := artRepo.GetItemArtwork("al", "alA", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(iaA.Source).To(Equal("folder")) + Expect(filepath.ToSlash(iaA.SourcePath)).To(HaveSuffix("album-a/cover.jpg")) + Expect(iaA.RefMtime).To(Equal(int64(1000))) + + // Poison the shared row's blurhash: the second item must dedup on hash, not re-decode. + poisoned := artRepo.Data[iaA.Hash] + poisoned.BlurHash = "SENTINEL" + artRepo.Data[iaA.Hash] = poisoned + + folderRepo.result = []model.Folder{{Path: "album-b", ImageFiles: []string{"cover.jpg"}}} + Expect(processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alB"})).To(Equal(outcomeFound)) + iaB, err := artRepo.GetItemArtwork("al", "alB", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(iaB.Hash).To(Equal(iaA.Hash)) + Expect(filepath.ToSlash(iaB.SourcePath)).To(HaveSuffix("album-b/cover.jpg")) + Expect(iaB.RefMtime).To(Equal(int64(2000))) + + // The first item's provenance survives the second item processing identical bytes. + iaAafter, err := artRepo.GetItemArtwork("al", "alA", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(filepath.ToSlash(iaAafter.SourcePath)).To(HaveSuffix("album-a/cover.jpg")) + Expect(iaAafter.RefMtime).To(Equal(int64(1000))) + + // One shared artwork row, and dedup preserved it untouched. + Expect(artRepo.Data).To(HaveLen(1)) + reused, err := artRepo.GetImage(iaA.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(reused.BlurHash).To(Equal("SENTINEL")) + }) + + It("decode failure on found bytes: fails without writing state", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra1_test.jpg") + Expect(os.WriteFile(imgPath, []byte("not actually an image"), 0600)).To(Succeed()) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio", UploadedImage: "ra1_test.jpg"}} + ds.MockedRadio = radioRepo + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork("ra", "ra1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("oversized read: a resolved image larger than the cap fails without writing state", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "radio", "big_test.jpg") + f, err := os.Create(imgPath) + Expect(err).ToNot(HaveOccurred()) + Expect(f.Truncate(maxImageBytes + 1)).To(Succeed()) + Expect(f.Close()).To(Succeed()) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"big": {ID: "big", Name: "Radio", UploadedImage: "big_test.jpg"}} + ds.MockedRadio = radioRepo + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "big"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err = artRepo.GetItemArtwork("ra", "big", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("decompression bomb: rejects huge declared dimensions before the full decode", func() { + data := pngHeaderWithDims(50000, 50000) // 2.5 gigapixels, far above the cap + _, err := decodeArtwork(ctx, "bomb", data) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dimensions")) + }) + + It("store write failure: fails without writing state", func() { + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al7", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + folderRepo.result = nil + + // A store root that is a plain file makes every MkdirAll under it fail. + blockedRoot := filepath.Join(GinkgoT().TempDir(), "not-a-dir") + Expect(os.WriteFile(blockedRoot, []byte("x"), 0600)).To(Succeed()) + deps.store = NewImageStore(blockedRoot) + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork("al", "al7", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) +}) diff --git a/core/artwork/prune.go b/core/artwork/prune.go new file mode 100644 index 000000000..35eddb525 --- /dev/null +++ b/core/artwork/prune.go @@ -0,0 +1,86 @@ +package artwork + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// pruneMinAge guards the window between artwork insert and item_artwork upsert. +const pruneMinAge = time.Hour + +func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { + repo := ds.Artwork(ctx) + + purged, err := repo.PurgeDanglingItemArtwork() + if err != nil { + return err + } + if purged > 0 { + log.Info(ctx, "Prune: purged dangling item artwork state", "count", purged) + } + + // Queue rows for deleted entities would otherwise retry forever (Get -> not found -> failed). + queuePurged, err := ds.ArtworkQueue(ctx).PurgeDangling() + if err != nil { + return err + } + if queuePurged > 0 { + log.Info(ctx, "Prune: purged dangling artwork queue rows", "count", queuePurged) + } + + // One grace cutoff for both the DB orphan check and the file sweep: files younger + // than the window may belong to acquisitions whose rows aren't committed yet. + cutoff := time.Now().Add(-pruneMinAge) + candidates, err := repo.GetOrphanHashes(cutoff) + if err != nil { + return err + } + if len(candidates) > 0 { + arts, err := repo.GetImages(candidates) + if err != nil { + return err + } + if err := repo.DeleteOrphans(cutoff, candidates); err != nil { + return err + } + // DeleteOrphans may spare candidates reacquired since the snapshot; only remove files + // for rows actually gone (absent from the post-delete re-read). + survivors, err := repo.GetImages(candidates) + if err != nil { + return err + } + removed := 0 + for _, h := range candidates { + if _, ok := survivors[h]; ok { + continue + } + // A spared fresh file is at worst a stray a later sweep reclaims; + // Worker.RunPrune serializes prune against in-flight acquisitions. + if err := store.Remove(h, arts[h].Mime, cutoff); err != nil { + log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err) + } + removed++ + } + log.Info(ctx, "Prune: removed orphan artwork", "count", removed) + } + + mimes, err := repo.GetAllMimes() + if err != nil { + return err + } + removed, err := store.Sweep(cutoff, func(hash, ext string) bool { + // A known hash under a stale extension is a superseded mime variant — reclaim it. + m, ok := mimes[hash] + return ok && ext == extForMime(m) + }) + if err != nil { + return err + } + if removed > 0 { + log.Info(ctx, "Prune: swept stray artwork files", "count", removed) + } + return nil +} diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go new file mode 100644 index 000000000..646ad7440 --- /dev/null +++ b/core/artwork/prune_test.go @@ -0,0 +1,254 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "time" + + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type flakyGetArtworkRepo struct { + *tests.MockArtworkRepo +} + +func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) { + return nil, errors.New("db locked") +} + +var _ = Describe("Prune", func() { + var ds *tests.MockDataStore + var store *ImageStore + var awRepo *tests.MockArtworkRepo + + BeforeEach(func() { + ds = &tests.MockDataStore{} + awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo) + store = NewImageStore(GinkgoT().TempDir()) + }) + + // PutImage refreshes created_at like the SQL repo, so fixtures are aged directly. + ageArtwork := func(h string, t time.Time) { + a := awRepo.Data[h] + a.CreatedAt = t + awRepo.Data[h] = a + } + + It("purges dangling item_artwork state for gone entities, summed across kinds", func() { + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "gone-album", ImageType: model.ImageTypePrimary})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "gone-artist", ImageType: model.ImageTypePrimary})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "live-artist", ImageType: model.ImageTypePrimary})).To(Succeed()) + awRepo.ExistingIDs = map[string]map[string]bool{ + "al": {}, + "ar": {"live-artist": true}, + } + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetItemArtwork("al", "gone-album", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = awRepo.GetItemArtwork("ar", "gone-artist", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = awRepo.GetItemArtwork("ar", "live-artist", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + }) + + It("purges dangling artwork_queue rows for gone entities", func() { + queueRepo := tests.CreateMockArtworkQueueRepo() + Expect(queueRepo.Enqueue( + model.ArtworkQueueItem{ItemKind: "al", ItemID: "gone-album", ImageType: model.ImageTypePrimary}, + model.ArtworkQueueItem{ItemKind: "al", ItemID: "live-album", ImageType: model.ImageTypePrimary}, + )).To(Succeed()) + queueRepo.ExistingIDs = map[string]map[string]bool{"al": {"live-album": true}} + ds.MockedArtworkQueue = queueRepo + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + Expect(findQueued(queueRepo, "al", "gone-album")).To(BeNil()) + Expect(findQueued(queueRepo, "al", "live-album")).ToNot(BeNil()) + }) + + It("deletes orphan rows and their store files, keeps referenced ones", func() { + data := []byte("orphan-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, old) + awRepo.OrphanHashes = []string{h} + + kept := []byte("kept-bytes") + hk, _ := HashImage(bytes.NewReader(kept)) + Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = store.Open(h, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(hk, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("spares a candidate reacquired between snapshot and delete", func() { + data := []byte("reacquired-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, time.Now().Add(-2*time.Hour)) + awRepo.OrphanHashes = []string{h} + // Reacquisition: an item now references the hash the snapshot flagged as orphan. + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", + ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).ToNot(HaveOccurred()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() { + data := []byte("fresh-reacquired-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + // Reacquisition refreshed created_at after the snapshot; still unreferenced. + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + awRepo.OrphanHashes = []string{h} + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).ToNot(HaveOccurred()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("spares an orphan file freshly touched by an overlapping acquisition", func() { + data := []byte("racing-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, time.Now().Add(-2*time.Hour)) + awRepo.OrphanHashes = []string{h} + // The row is legitimately orphaned, but a concurrent acquisition just touched the + // file's mtime (duplicate Write) and is about to commit a row referencing it. + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("sweeps store files that have no artwork row", func() { + stray := []byte("no-row-bytes") + h, _ := HashImage(bytes.NewReader(stray)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := store.Open(h, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("sweeps an obsolete mime variant of a reacquired hash", func() { + data := []byte("variant-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + // The row records the current mime; the .png file is a superseded variant. + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := store.Open(h, "image/png") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("warns and continues past a store.Remove failure instead of aborting the loop", func() { + tests.SkipOnWindows("uses Unix file permission bits") + if os.Geteuid() == 0 { + Skip("read-only dir cannot block root (e.g. tests in a container)") + } + old := time.Now().Add(-2 * time.Hour) + + blocked := []byte("blocked-bytes") + hb, _ := HashImage(bytes.NewReader(blocked)) + Expect(store.Write(hb, "image/jpeg", bytes.NewReader(blocked))).To(Succeed()) + Expect(os.Chtimes(store.path(hb, "image/jpeg"), old, old)).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hb, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(hb, old) + + good := []byte("good-bytes") + hg, _ := HashImage(bytes.NewReader(good)) + Expect(store.Write(hg, "image/jpeg", bytes.NewReader(good))).To(Succeed()) + Expect(os.Chtimes(store.path(hg, "image/jpeg"), old, old)).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hg, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(hg, old) + + // A read-only shard directory makes os.Remove fail (EACCES) for hb's file only. + shardDir := filepath.Dir(store.path(hb, "image/jpeg")) + Expect(os.Chmod(shardDir, 0500)).To(Succeed()) + DeferCleanup(func() { _ = os.Chmod(shardDir, 0755) }) + + // hb (blocked) is processed first: if store.Remove's failure aborted the loop + // instead of warning and continuing, hg would never be reached. + awRepo.OrphanHashes = []string{hb, hg} + + // Prune still errors: Sweep independently revisits hb's leftover file and, + // unlike the loop below, has no warn-and-continue fallback of its own. + err := Prune(context.Background(), ds, store) + Expect(err).To(HaveOccurred()) + + // hg: reached and fully pruned despite being queued after the failing hb - + // proof the loop didn't return/break on the first Remove error. + _, err = awRepo.GetImage(hg) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = store.Open(hg, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + + // hb: row still purged (DeleteOrphans doesn't depend on file removal), but the + // file itself survives since store.Remove failed and only warned. + _, err = awRepo.GetImage(hb) + Expect(err).To(MatchError(model.ErrNotFound)) + rc, err := store.Open(hb, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("never sweeps files on a transient DB error", func() { + ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()} + + data := []byte("live-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).ToNot(Succeed()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) +}) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go new file mode 100644 index 000000000..8a62b2999 --- /dev/null +++ b/core/artwork/resolve.go @@ -0,0 +1,423 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + "image/draw" + "image/png" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/model" +) + +// resolution is one attempted acquisition outcome for an entity. +type resolution struct { + reader io.ReadCloser // nil when no source yielded an image + source string // model.ItemArtwork.Source value: "folder", "embedded", "external", "upload", "generated" + sourcePath string // backing library/upload file (folder/upload: the image; embedded: the audio file); "" otherwise + refMtime int64 // mtime of sourcePath at resolution time; 0 when no sourcePath + // external source errored/timed out. With no reader: forces failed (never absent). + // On a hit: a higher-priority external step failed—serve this, but retry later. + extError bool +} + +// extGateFunc is an alias for the external-step wrapper the worker injects (rate +// limiter + circuit breaker); resolveItem defaults to a plain passthrough. +type extGateFunc = func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) + +func passthroughExtGate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + return f() +} + +// resolveItem walks the kind's priority chain and returns the first hit. +func resolveItem(ctx context.Context, ds model.DataStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg, item model.ArtworkQueueItem, extGate extGateFunc) (resolution, error) { + if extGate == nil { + extGate = passthroughExtGate + } + switch item.ItemKind { + case "al": + return resolveAlbum(ctx, ds, prov, ffmpeg, item.ItemID, extGate) + case "ar": + return resolveArtist(ctx, ds, prov, ffmpeg, item.ItemID, extGate) + case "pl": + return resolvePlaylist(ctx, ds, prov, ffmpeg, item.ItemID, extGate) + case "ra": + return resolveRadio(ctx, ds, item.ItemID) + default: + return resolution{}, fmt.Errorf("resolveItem: kind %q is not resolvable by the worker", item.ItemKind) + } +} + +// resolveAlbum ports the folder/embedded/external selection from +// reader_album.go, walking conf.Server.CoverArtPriority. +func resolveAlbum(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, albumID string, extGate extGateFunc) (resolution, error) { + al, err := ds.Album(ctx).Get(albumID) + if err != nil { + return resolution{}, err + } + _, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, *al) + if err != nil { + return resolution{}, err + } + lib, err := loadLibraryView(ctx, ds, al.LibraryID) + if err != nil { + return resolution{}, err + } + + var extErr bool + for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.CoverArtPriority), ",") { + pattern = strings.TrimSpace(pattern) + switch { + case pattern == "embedded": + if res, ok := resolveEmbedded(ctx, lib, ffm, al.EmbedArtPath); ok { + res.extError = extErr + return res, nil + } + case pattern == "external": + if res, ok, isErr := resolveExternalStep(extGate, fromAlbumExternalSource(ctx, *al, prov)); ok { + return res, nil + } else if isErr { + extErr = true + } + case len(imgFiles) > 0: + if res, ok := resolveFolderFile(ctx, lib, imgFiles, pattern); ok { + res.extError = extErr + return res, nil + } + } + } + return resolution{extError: extErr}, nil +} + +// resolveArtist ports the upload/folder/external selection from +// reader_artist.go: upload always wins, then conf.Server.ArtistArtPriority. +func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, artistID string, extGate extGateFunc) (resolution, error) { + ar, err := ds.Artist(ctx).Get(artistID) + if err != nil { + return resolution{}, err + } + if res, ok := resolveLocalFile(ar.UploadedImagePath(), "upload"); ok { + return res, nil + } + + // Only consider albums where the artist is the sole album artist, same as reader_artist.go. + als, err := ds.Album(ctx).GetAll(model.QueryOptions{ + Filters: squirrel.And{ + squirrel.Eq{"album_artist_id": artistID}, + squirrel.Eq{"json_array_length(participants, '$.albumartist')": 1}, + }, + }) + if err != nil { + return resolution{}, err + } + albumPaths, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, als...) + if err != nil { + return resolution{}, err + } + artistFolder, _, err := loadArtistFolder(ctx, ds, als, albumPaths) + if err != nil { + return resolution{}, err + } + var lib libraryView + if len(als) > 0 { + lib, err = loadLibraryView(ctx, ds, als[0].LibraryID) + if err != nil { + return resolution{}, err + } + } + + var extErr bool + for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.ArtistArtPriority), ",") { + pattern = strings.TrimSpace(pattern) + switch { + case pattern == "external": + if res, ok, isErr := resolveExternalStep(extGate, fromArtistExternalResult(ctx, *ar, prov)); ok { + return res, nil + } else if isErr { + extErr = true + } + case pattern == "image-folder": + if res, ok := resolveArtistImageFolder(ar); ok { + res.extError = extErr + return res, nil + } + case strings.HasPrefix(pattern, "album/"): + if lib.FS == nil { + continue + } + if res, ok := resolveFolderFile(ctx, lib, imgFiles, strings.TrimPrefix(pattern, "album/")); ok { + res.extError = extErr + return res, nil + } + default: + if lib.FS == nil || artistFolder == "" { + continue + } + if res, ok := resolveArtistFolderPattern(ctx, lib, artistFolder, pattern); ok { + res.extError = extErr + return res, nil + } + } + } + return resolution{extError: extErr}, nil +} + +// resolvePlaylist ports reader_playlist.go's chain: uploaded image, sidecar, +// ExternalImageURL, then the generated 2x2 grid sourced through resolveAlbum. +func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, playlistID string, extGate extGateFunc) (resolution, error) { + pl, err := ds.Playlist(ctx).Get(playlistID) + if err != nil { + return resolution{}, err + } + + var extErr bool + if res, ok := resolveLocalFile(pl.UploadedImagePath(), "upload"); ok { + return res, nil + } + if res, ok := resolveLocalFile(findPlaylistSidecarPath(ctx, pl.Path), "folder"); ok { + return res, nil + } + if res, ok, isErr := resolveExternalStep(extGate, fromPlaylistExternalSource(ctx, *pl)); ok { + return res, nil + } else if isErr { + extErr = true + } + + albumIDs, err := ds.Playlist(ctx).Tracks(pl.ID, false).GetAlbumIDs(model.QueryOptions{Max: 4, Sort: "random()"}) + if err != nil { + return resolution{}, err + } + + var tiles []image.Image + var tileErr error // first internal (non-external) tile failure, e.g. album deleted mid-flight + for _, albumID := range albumIDs { + res, err := resolveAlbum(ctx, ds, prov, ffm, albumID, extGate) + if err != nil { + if tileErr == nil { + tileErr = err + } + continue + } + if res.extError { + extErr = true + } + if res.reader == nil { + continue + } + tile, decErr := decodeTile(res.reader) + res.reader.Close() + if decErr == nil { + tiles = append(tiles, tile) + } + if len(tiles) == 4 { + break + } + } + if len(tiles) == 0 { + // A tile-level failure must never resolve as a clean absent: propagate + // internal errors, and force extError for external ones. + if tileErr != nil { + return resolution{}, fmt.Errorf("resolvePlaylist: sampled album art failed: %w", tileErr) + } + return resolution{extError: extErr}, nil + } + // Grow to 4 tiles by repeating what we have, mirroring reader_playlist.go's loadTiles. + switch len(tiles) { + case 2: + tiles = append(tiles, tiles[1], tiles[0]) + case 3: + tiles = append(tiles, tiles[0]) + } + r, err := assembleTiles(tiles) + if err != nil { + return resolution{extError: extErr}, nil //nolint:nilerr // encode failure is a soft "no image", not a resolveItem error + } + return resolution{reader: r, source: "generated", extError: extErr}, nil +} + +// resolveRadio ports reader_radio.go: only an uploaded image, no fallback. +func resolveRadio(ctx context.Context, ds model.DataStore, radioID string) (resolution, error) { + r, err := ds.Radio(ctx).Get(radioID) + if err != nil { + return resolution{}, err + } + res, _ := resolveLocalFile(r.UploadedImagePath(), "upload") + return res, nil +} + +// resolveExternalStep runs an external sourceFunc through extGate, shared by +// resolveAlbum and resolveArtist. ok reports a hit; extErr reports a +// non-not-found error (a not-found is a definitive "no", not a failure). +func resolveExternalStep(extGate extGateFunc, sf func() (io.ReadCloser, string, error)) (res resolution, ok bool, extErr bool) { + r, path, err := extGate(sf) + if r != nil { + return resolution{reader: r, source: "external", sourcePath: path}, true, false + } + return resolution{}, false, err != nil && !errors.Is(err, model.ErrNotFound) +} + +// fromPlaylistExternalSource mirrors reader_playlist.go's ExternalImageURL step: +// a remote URL (gated) when M3U external art is enabled, else a local file path. +func fromPlaylistExternalSource(ctx context.Context, pl model.Playlist) sourceFunc { + return func() (io.ReadCloser, string, error) { + imgURL := pl.ExternalImageURL + if imgURL == "" { + return nil, "", nil + } + parsed, err := url.Parse(imgURL) + if err != nil { + return nil, "", err + } + if parsed.Scheme == "http" || parsed.Scheme == "https" { + if !conf.Server.EnableM3UExternalAlbumArt { + return nil, "", nil + } + return fetchPlaylistImageURL(ctx, parsed) + } + // A missing/unreadable local file is a definitive miss, not a transient + // failure to retry: swallow the open error and fall through to the grid. + r, path, _ := fromLocalFile(imgURL)() + return r, path, nil + } +} + +// Like sources.go's fromURL but maps 404/410 to ErrNotFound (definitive), so a stale M3U +// cover URL falls through to the grid instead of retrying forever and tripping the breaker. +func fetchPlaylistImageURL(ctx context.Context, imageURL *url.URL) (io.ReadCloser, string, error) { + hc := http.Client{Timeout: 5 * time.Second} + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, imageURL.String(), nil) + req.Header.Set("User-Agent", consts.HTTPUserAgent) + resp, err := hc.Do(req) //nolint:gosec + if err != nil { + return nil, "", err + } + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + resp.Body.Close() + return nil, "", model.ErrNotFound + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, "", fmt.Errorf("error retrieving artwork from %s: %s", imageURL, resp.Status) + } + return resp.Body, imageURL.String(), nil +} + +func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, embedRel string) (resolution, bool) { + if embedRel == "" { + return resolution{}, false + } + abs := lib.Abs(embedRel) + for _, sf := range []sourceFunc{fromTag(ctx, lib.FS, embedRel), fromFFmpegTag(ctx, ffm, abs)} { + if r, _, _ := sf(); r != nil { + return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtimeViaFS(lib.FS, embedRel)}, true + } + } + return resolution{}, false +} + +func resolveFolderFile(ctx context.Context, lib libraryView, imgFiles []string, pattern string) (resolution, bool) { + r, path, _ := fromExternalFile(ctx, lib.FS, imgFiles, pattern)() + if r == nil { + return resolution{}, false + } + return resolution{reader: r, source: "folder", sourcePath: lib.Abs(path), refMtime: mtimeViaFS(lib.FS, path)}, true +} + +func resolveArtistImageFolder(ar *model.Artist) (resolution, bool) { + folder := conf.Server.ArtistImageFolder + if folder == "" { + return resolution{}, false + } + return resolveLocalFile(findImageInArtistFolder(folder, ar.MbzArtistID, ar.Name), "folder") +} + +func resolveArtistFolderPattern(ctx context.Context, lib libraryView, artistFolder, pattern string) (resolution, bool) { + r, path, _ := fromArtistFolder(ctx, lib.FS, lib.absRoot, artistFolder, pattern)() + if r == nil { + return resolution{}, false + } + return resolution{reader: r, source: "folder", sourcePath: path, refMtime: mtimeOf(path)}, true +} + +// resolveLocalFile opens an absolute path directly (uploads, image-folder). A +// missing or unreadable path is "no source", not an error. +func resolveLocalFile(path, source string) (resolution, bool) { + if path == "" { + return resolution{}, false + } + f, err := os.Open(path) + if err != nil { + return resolution{}, false + } + return resolution{reader: f, source: source, sourcePath: path, refMtime: mtimeOf(path)}, true +} + +func mtimeOf(path string) int64 { + info, err := os.Stat(path) + if err != nil { + return 0 + } + return info.ModTime().Unix() +} + +// mtimeViaFS stats through the library FS instead of a joined absolute path, +// since library roots in tests may not be real OS paths (e.g. testfile://). +func mtimeViaFS(fsys fs.FS, name string) int64 { + if fsys == nil || name == "" { + return 0 + } + info, err := fs.Stat(fsys, name) + if err != nil { + return 0 + } + return info.ModTime().Unix() +} + +// decodeTile and assembleTiles mirror playlistArtworkReader's createTile/ +// createTiledImage, reusing the same rect/fillCenter cropping helpers. +// decodeTile runs on every sampled album's resolved bytes before processItem's +// own maxImageBytes/maxImagePixels guards apply, so it enforces them itself too. +func decodeTile(r io.ReadCloser) (image.Image, error) { + data, err := readCapped(r) + if err != nil { + return nil, err + } + img, _, err := decodeCapped(data) + if err != nil { + return nil, err + } + return fillCenter(img, tileSize/2, tileSize/2), nil +} + +func assembleTiles(tiles []image.Image) (io.ReadCloser, error) { + buf := new(bytes.Buffer) + var err error + if len(tiles) == 4 { + rgba := image.NewRGBA(image.Rectangle{Max: image.Point{X: tileSize - 1, Y: tileSize - 1}}) + draw.Draw(rgba, rect(0), tiles[0], image.Point{}, draw.Src) + draw.Draw(rgba, rect(1), tiles[1], image.Point{}, draw.Src) + draw.Draw(rgba, rect(2), tiles[2], image.Point{}, draw.Src) + draw.Draw(rgba, rect(3), tiles[3], image.Point{}, draw.Src) + err = png.Encode(buf, rgba) + } else { + err = png.Encode(buf, tiles[0]) + } + if err != nil { + return nil, err + } + return io.NopCloser(buf), nil +} diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go new file mode 100644 index 000000000..c64ee2b53 --- /dev/null +++ b/core/artwork/resolve_test.go @@ -0,0 +1,562 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "image" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// fakeExternalProvider is a minimal external.Provider stub for resolve_test.go; +// only AlbumImage/ArtistImage are exercised by the resolvers. +type fakeExternalProvider struct { + external.Provider + albumImage func(ctx context.Context, id string) (*url.URL, error) + artistImage func(ctx context.Context, id string) (*url.URL, error) +} + +func (f *fakeExternalProvider) AlbumImage(ctx context.Context, id string) (*url.URL, error) { + if f.albumImage != nil { + return f.albumImage(ctx, id) + } + return nil, model.ErrNotFound +} + +func (f *fakeExternalProvider) ArtistImage(ctx context.Context, id string) (*url.URL, error) { + if f.artistImage != nil { + return f.artistImage(ctx, id) + } + return nil, model.ErrNotFound +} + +func (f *fakeExternalProvider) ArtistImageResult(ctx context.Context, id string) (*url.URL, error) { + return f.ArtistImage(ctx, id) +} + +var _ = Describe("resolveItem", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + folderRepo *fakeFolderRepo + libRepo *tests.MockLibraryRepo + ffm *tests.MockFFmpeg + prov *fakeExternalProvider + repoRoot string + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + folderRepo = &fakeFolderRepo{} + libRepo = &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + ffm = tests.NewMockFFmpeg("") + prov = &fakeExternalProvider{} + ds = &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + } + }) + + Describe("kind dispatch", func() { + It("returns an error for kinds the worker never enqueues", func() { + _, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "mf", ItemID: "x"}, nil) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("album", func() { + BeforeEach(func() { + conf.Server.CoverArtPriority = "cover.jpg, embedded" + ds.MockedAlbum = tests.CreateMockAlbumRepo() + }) + + It("resolves folder art from the library FS", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}, + }) + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg")) + Expect(res.refMtime).To(BeNumerically(">", 0)) + Expect(res.extError).To(BeFalse()) + }) + + It("falls back to embedded art when no folder image matches", func() { + folderRepo.result = nil + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("embedded")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3")) + Expect(res.refMtime).To(BeNumerically(">", 0)) + }) + + It("sets extError when the external source errors without being not-found", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al3", Name: "Album"}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + }) + + It("does not set extError when the external source reports not-found", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al4", Name: "Album"}, + }) + // prov.albumImage left nil -> fakeExternalProvider returns model.ErrNotFound + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeFalse()) + }) + + It("carries extError onto a fallback folder hit after a transient external failure", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al6", Name: "Album", FolderIDs: []string{"f1"}}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(res.extError).To(BeTrue()) + }) + + It("does not carry extError onto a fallback folder hit after a definitive external not-found", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al7", Name: "Album", FolderIDs: []string{"f1"}}, + }) + // prov.albumImage left nil -> fakeExternalProvider returns model.ErrNotFound + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(res.extError).To(BeFalse()) + }) + + It("routes the external step through a custom extGate", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al5", Name: "Album"}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("boom") + } + var extGateCalls int + extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + extGateCalls++ + return f() + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"}, extGate) + Expect(err).ToNot(HaveOccurred()) + Expect(res.extError).To(BeTrue()) + Expect(extGateCalls).To(Equal(1)) + }) + }) + + Describe("artist", func() { + It("resolves the uploaded image before any priority chain lookup", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "artist"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "artist", "ar1_test.jpg") + Expect(os.WriteFile(imgPath, []byte("uploaded artist image"), 0600)).To(Succeed()) + + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar1", Name: "Artist", UploadedImage: "ar1_test.jpg"}}) + ds.MockedArtist = artistRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("upload")) + Expect(res.sourcePath).To(Equal(imgPath)) + }) + + It("falls through to the ArtistArtPriority chain when there is no upload", func() { + conf.Server.ArtistArtPriority = "album/artist.*" + folderRepo.result = []model.Folder{{ + LibraryPath: testFileLibPath(repoRoot), + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"artist.png"}, + }} + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar2", Name: "Artist"}}) + ds.MockedArtist = artistRepo + ds.MockedAlbum = tests.CreateMockAlbumRepo() + ds.MockedAlbum.(*tests.MockAlbumRepo).All = model.Albums{ + {ID: "al9", Name: "Album", LibraryID: 0, FolderIDs: []string{"f1"}}, + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/artist.png")) + }) + + It("sets extError when the external source errors without being not-found", func() { + conf.Server.ArtistArtPriority = "external" + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar3", Name: "Artist"}}) + ds.MockedArtist = artistRepo + prov.artistImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar3"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + }) + + It("does not set extError when the external source reports not-found", func() { + conf.Server.ArtistArtPriority = "external" + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar4", Name: "Artist"}}) + ds.MockedArtist = artistRepo + // prov.artistImage left nil -> fakeExternalProvider returns model.ErrNotFound + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar4"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeFalse()) + }) + + It("routes the external step through a custom extGate", func() { + conf.Server.ArtistArtPriority = "external" + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar5", Name: "Artist"}}) + ds.MockedArtist = artistRepo + prov.artistImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("boom") + } + var extGateCalls int + extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + extGateCalls++ + return f() + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar5"}, extGate) + Expect(err).ToNot(HaveOccurred()) + Expect(res.extError).To(BeTrue()) + Expect(extGateCalls).To(Equal(1)) + }) + }) + + Describe("radio", func() { + It("yields an empty resolution when there is no uploaded image", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio"}} + ds.MockedRadio = radioRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal(resolution{})) + }) + + It("resolves the uploaded image when set", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra2_test.jpg") + Expect(os.WriteFile(imgPath, []byte("uploaded radio image"), 0600)).To(Succeed()) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"ra2": {ID: "ra2", Name: "Radio", UploadedImage: "ra2_test.jpg"}} + ds.MockedRadio = radioRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("upload")) + Expect(res.sourcePath).To(Equal(imgPath)) + }) + }) + + Describe("playlist", func() { + BeforeEach(func() { + conf.Server.CoverArtPriority = "cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum = tests.CreateMockAlbumRepo() + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "t1", Name: "T1", FolderIDs: []string{"f1"}}, + {ID: "t2", Name: "T2", FolderIDs: []string{"f1"}}, + {ID: "t3", Name: "T3", FolderIDs: []string{"f1"}}, + {ID: "t4", Name: "T4", FolderIDs: []string{"f1"}}, + }) + }) + + DescribeTable("yields a generated grid from up to 4 album tiles", + func(albumIDs []string, expectedSize int) { + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl1", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: albumIDs} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("generated")) + + img, format, err := image.Decode(res.reader) + Expect(err).ToNot(HaveOccurred()) + Expect(format).To(Equal("png")) + Expect(img.Bounds().Dx()).To(Equal(expectedSize)) + Expect(img.Bounds().Dy()).To(Equal(expectedSize)) + }, + // tileSize-1: the 4-tile canvas is built as [0, tileSize-1], matching + // reader_playlist.go's createTiledImage exactly. + Entry("1 album -> single tile", []string{"t1"}, tileSize/2), + Entry("2 albums -> duplicated to 4 tiles", []string{"t1", "t2"}, tileSize-1), + Entry("3 albums -> duplicated to 4 tiles", []string{"t1", "t2", "t3"}, tileSize-1), + Entry("4 albums -> full grid", []string{"t1", "t2", "t3", "t4"}, tileSize-1), + ) + + It("resolves the uploaded image before the generated grid", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "playlist"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "playlist", "plu_test.jpg") + Expect(os.WriteFile(imgPath, []byte("uploaded playlist image"), 0600)).To(Succeed()) + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plu", Name: "Playlist", UploadedImage: "plu_test.jpg"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plu"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("upload")) + Expect(res.sourcePath).To(Equal(imgPath)) + }) + + It("resolves a sidecar image next to the playlist file before the grid", func() { + plDir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(plDir, "list.m3u"), []byte("#EXTM3U"), 0600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(plDir, "list.jpg"), []byte("sidecar image"), 0600)).To(Succeed()) + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pls", Name: "Playlist", Path: filepath.Join(plDir, "list.m3u")}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pls"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("list.jpg")) + }) + + It("routes ExternalImageURL through extGate and sets extError on transient failure", func() { + conf.Server.EnableM3UExternalAlbumArt = true + folderRepo.result = nil // no grid tiles, so the external failure is what surfaces + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "ple", Name: "Playlist", ExternalImageURL: "http://example.com/cover.jpg"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + var extGateCalls int + extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + extGateCalls++ + return nil, "", errors.New("network down") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "ple"}, extGate) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + Expect(extGateCalls).To(Equal(1)) + }) + + It("treats a missing local ExternalImageURL as a definitive miss, not extError", func() { + folderRepo.result = nil // no grid tiles, so the local-file miss is what surfaces + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plm", Name: "Playlist", ExternalImageURL: "/nonexistent/path/cover.jpg"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plm"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeFalse()) + }) + + It("treats an ExternalImageURL 404 as a definitive miss and falls through to the grid", func() { + conf.Server.EnableM3UExternalAlbumArt = true + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl404", Name: "Playlist", ExternalImageURL: srv.URL}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl404"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("generated")) + Expect(res.extError).To(BeFalse()) + }) + + It("treats an ExternalImageURL 500 as a transient failure and sets extError", func() { + conf.Server.EnableM3UExternalAlbumArt = true + folderRepo.result = nil // no grid tiles, so the external failure is what surfaces + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl500", Name: "Playlist", ExternalImageURL: srv.URL}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl500"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + }) + + It("yields an empty resolution when no album has art", func() { + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "empty1", Name: "Empty"}, + }) + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl2", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"empty1"}} + ds.MockedPlaylist = plRepo + folderRepo.result = nil + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.source).To(BeEmpty()) + }) + + It("skips a grid tile whose declared dimensions are a decompression bomb", func() { + // End-to-end regression: a bomb-declaring tile must not break the grid. + libRoot := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(libRoot, "bomb"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(libRoot, "bomb", "cover.jpg"), pngHeaderWithDims(50000, 50000), 0600)).To(Succeed()) + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + folderRepo.result = []model.Folder{{Path: "bomb", ImageFiles: []string{"cover.jpg"}}} + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plbomb", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plbomb"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.source).To(BeEmpty()) + }) + + It("does not resolve as absent when every sampled album fails to resolve", func() { + // "missing1"/"missing2" are not in MockAlbumRepo's data, so resolveAlbum + // returns a genuine (non-external) error for every sampled tile. + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl3", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"missing1", "missing2"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl3"}, nil) + Expect(err).To(HaveOccurred()) + Expect(res).To(Equal(resolution{})) + }) + }) +}) + +// decodeTile runs on every sampled album's resolved bytes before processItem's +// own guards apply, so it must enforce the same caps independently. +var _ = Describe("decodeTile", func() { + It("rejects a decompression bomb before the full decode", func() { + data := pngHeaderWithDims(50000, 50000) // 2.5 gigapixels, far above the cap + _, err := decodeTile(io.NopCloser(bytes.NewReader(data))) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dimensions")) + }) + + It("rejects a tile larger than the size cap", func() { + data := bytes.Repeat([]byte{0}, maxImageBytes+1) + _, err := decodeTile(io.NopCloser(bytes.NewReader(data))) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/artwork/sources.go b/core/artwork/sources.go index 04a9257fb..d4ad0a2a7 100644 --- a/core/artwork/sources.go +++ b/core/artwork/sources.go @@ -198,6 +198,18 @@ func fromArtistExternalSource(ctx context.Context, ar model.Artist, provider ext } } +// fromArtistExternalResult is the worker's artist external step: via ArtistImageResult a +// transient agent failure surfaces as an error (extError) rather than settling as absent. +func fromArtistExternalResult(ctx context.Context, ar model.Artist, provider external.Provider) sourceFunc { + return func() (io.ReadCloser, string, error) { + imageUrl, err := provider.ArtistImageResult(ctx, ar.ID) + if err != nil { + return nil, "", err + } + return fromURL(ctx, imageUrl) + } +} + func fromAlbumExternalSource(ctx context.Context, al model.Album, provider external.Provider) sourceFunc { return func() (io.ReadCloser, string, error) { imageUrl, err := provider.AlbumImage(ctx, al.ID) diff --git a/core/artwork/wire_providers.go b/core/artwork/wire_providers.go index 63231b54a..868ea8f56 100644 --- a/core/artwork/wire_providers.go +++ b/core/artwork/wire_providers.go @@ -8,4 +8,6 @@ var Set = wire.NewSet( NewArtwork, GetImageCache, NewCacheWarmer, + NewWorker, + ProvideImageStore, ) diff --git a/core/artwork/worker.go b/core/artwork/worker.go new file mode 100644 index 000000000..5ae40a6e7 --- /dev/null +++ b/core/artwork/worker.go @@ -0,0 +1,257 @@ +package artwork + +import ( + "context" + "errors" + "io" + "math" + "math/rand/v2" + "sync" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/core/auth" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "golang.org/x/time/rate" +) + +const ( + workerPollInterval = 5 * time.Second + backoffBase = 5 * time.Minute + backoffCap = 48 * time.Hour + breakerThreshold = 5 + breakerProbeAfter = time.Minute +) + +var errBreakerOpen = errors.New("artwork: external circuit breaker open") + +// Worker drains the artwork queue through processItem: the external step is rate-limited +// and circuit-broken, and prune is serialized against in-flight acquisitions via pruneMu. +type Worker struct { + deps workerDeps + limiter *rate.Limiter + breaker *breaker + pruneMu sync.RWMutex + wake chan struct{} + runCtx context.Context + + mu sync.Mutex + inFlight map[string]struct{} +} + +func NewWorker(ds model.DataStore, store *ImageStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg) *Worker { + rps := conf.Server.ArtworkExternalMaxRPS + limit := rate.Inf // 0 or negative disables the external throttle + if rps > 0 { + limit = rate.Limit(rps) + } + w := &Worker{ + deps: workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffmpeg}, + limiter: rate.NewLimiter(limit, max(1, rps)), + breaker: newBreaker(), + wake: make(chan struct{}, 1), + runCtx: context.Background(), + inFlight: map[string]struct{}{}, + } + w.deps.extGate = w.gate + return w +} + +// Run blocks draining the queue until ctx is cancelled. It exits cleanly with no +// leaked goroutines: each drain waits for its batch before the loop can return. +func (w *Worker) Run(ctx context.Context) error { + w.runCtx = ctx + concurrency := max(1, conf.Server.ArtworkWorkerConcurrency) + ticker := time.NewTicker(workerPollInterval) + defer ticker.Stop() + for { + n, err := w.drain(ctx, concurrency) + if err != nil && ctx.Err() == nil { + log.Warn(ctx, "artwork: worker drain failed", err) + } + if ctx.Err() != nil { + return nil + } + if n > 0 { + continue // keep draining while the queue has ready work + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + case <-w.wake: + } + } +} + +// Bump enqueues an item at the highest priority and wakes the drain loop. It is +// non-blocking: a wake already pending is enough. +func (w *Worker) Bump(kind, id string) { + item := model.ArtworkQueueItem{ + ItemKind: kind, + ItemID: id, + ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityBump, + } + if err := w.deps.ds.ArtworkQueue(context.Background()).Enqueue(item); err != nil { + log.Warn("artwork: could not bump queue item", "kind", kind, "id", id, err) + return + } + select { + case w.wake <- struct{}{}: + default: + } +} + +// RunPrune runs Prune under the worker's write lock, so no acquisition can place +// a file while orphans are being reclaimed. This is the only sanctioned prune path. +func (w *Worker) RunPrune(ctx context.Context) error { + w.pruneMu.Lock() + defer w.pruneMu.Unlock() + return Prune(ctx, w.deps.ds, w.deps.store) +} + +func (w *Worker) drain(ctx context.Context, concurrency int) (int, error) { + // Resolved per drain, not once in Run: the worker starts at boot, possibly before any + // admin exists, so a late-created admin is picked up on the next poll (private playlists). + ctx = auth.WithAdminUser(ctx, w.deps.ds) + batch, err := w.deps.ds.ArtworkQueue(ctx).DequeueBatch(2 * concurrency) + if err != nil { + return 0, err + } + items := w.claim(batch) + if len(items) == 0 { + return 0, nil + } + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + for _, item := range items { + sem <- struct{}{} + wg.Add(1) + go func(it model.ArtworkQueueItem) { + defer wg.Done() + defer func() { <-sem }() + defer w.release(it) + w.process(ctx, it) + }(item) + } + wg.Wait() + return len(items), nil +} + +func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) { + if item.ImageType == "" { + item.ImageType = model.ImageTypePrimary + } + w.pruneMu.RLock() + out := processItem(ctx, &w.deps, item) + w.pruneMu.RUnlock() + + queue := w.deps.ds.ArtworkQueue(ctx) + switch out { + case outcomeFound, outcomeAbsent: + // DeleteIfUnchanged, not Delete: a scan that re-enqueued this row mid-flight reset + // its retry_at, so the row survives here and the next drain re-resolves it. + if err := queue.DeleteIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt); err != nil { + log.Warn(ctx, "artwork: could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err) + } + case outcomeFoundStale, outcomeFailed: + // MarkFailedIfUnchanged, not MarkFailed: a scan that re-enqueued this row mid-flight reset + // retry_at, so stale backoff must not stomp its fresh, immediate eligibility. + retryAt := time.Now().Add(backoff(item.Attempts)) + if err := queue.MarkFailedIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt, retryAt); err != nil { + log.Warn(ctx, "artwork: could not reschedule failed queue item", "kind", item.ItemKind, "id", item.ItemID, err) + } + } +} + +// claim reserves items not already in flight, so a row appearing twice within a single +// batch is processed once. +func (w *Worker) claim(batch []model.ArtworkQueueItem) []model.ArtworkQueueItem { + w.mu.Lock() + defer w.mu.Unlock() + var out []model.ArtworkQueueItem + for _, it := range batch { + k := queueKey(it) + if _, busy := w.inFlight[k]; busy { + continue + } + w.inFlight[k] = struct{}{} + out = append(out, it) + } + return out +} + +func (w *Worker) release(it model.ArtworkQueueItem) { + w.mu.Lock() + delete(w.inFlight, queueKey(it)) + w.mu.Unlock() +} + +func queueKey(it model.ArtworkQueueItem) string { + return it.ItemKind + "|" + it.ItemID + "|" + it.ImageType +} + +// gate wraps the external step with the rate limiter and circuit breaker, matching +// extGateFunc so it can be injected via workerDeps.extGate. +func (w *Worker) gate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + if !w.breaker.allow() { + return nil, "", errBreakerOpen + } + if err := w.limiter.Wait(w.runCtx); err != nil { + return nil, "", err + } + r, path, err := f() + w.breaker.record(err) + return r, path, err +} + +// backoffFor returns min(5m×4^n, 48h) scaled by (1+jitter), with jitter in [-0.2, 0.2]. +func backoffFor(attempts int, jitter float64) time.Duration { + d := math.Min(float64(backoffBase)*math.Pow(4, float64(attempts)), float64(backoffCap)) + return time.Duration(d * (1 + jitter)) +} + +func backoff(attempts int) time.Duration { + return backoffFor(attempts, rand.Float64()*0.4-0.2) //nolint:gosec // retry jitter, not security-sensitive +} + +// breaker opens after breakerThreshold consecutive external errors and admits a +// single probe once breakerProbeAfter has elapsed; a success re-closes it. +type breaker struct { + mu sync.Mutex + failures int + openedAt time.Time +} + +func newBreaker() *breaker { return &breaker{} } + +func (b *breaker) allow() bool { + b.mu.Lock() + defer b.mu.Unlock() + if b.failures < breakerThreshold { + return true + } + if time.Since(b.openedAt) >= breakerProbeAfter { + b.openedAt = time.Now() // start a fresh probe window so only one caller passes + return true + } + return false +} + +func (b *breaker) record(err error) { + b.mu.Lock() + defer b.mu.Unlock() + // A not-found is a definitive answer, not a fault; only real errors trip the breaker. + if err == nil || errors.Is(err, model.ErrNotFound) { + b.failures = 0 + return + } + b.failures++ + if b.failures == breakerThreshold { + b.openedAt = time.Now() + } +} diff --git a/core/artwork/worker_soak_test.go b/core/artwork/worker_soak_test.go new file mode 100644 index 000000000..355ecb7ca --- /dev/null +++ b/core/artwork/worker_soak_test.go @@ -0,0 +1,150 @@ +package artwork + +import ( + "context" + "io" + "os" + "runtime" + "testing" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// soakCycles is deliberately >2000: this is a leak regression guard, not a +// performance benchmark, so it favors a stable signal over raw speed. +const soakCycles = 2200 + +var _ = Describe("Worker soak", func() { + // Runs processItem over many cycles across a mix of sources, asserting + // goroutines/heap plateau instead of growing unbounded (a leak guard). Skipped under -short. + It("does not leak goroutines, heap, or fds over many acquisition cycles", func() { + if testing.Short() { + Skip("skipping soak test in short mode") + } + DeferCleanup(configtest.SetupConfig()) + + repoRoot, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + folderRepo := &fakeFolderRepo{result: []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }}} + ffm := tests.NewMockFFmpeg("") + prov := &fakeExternalProvider{} + artRepo := tests.CreateMockArtworkRepo() + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{ + {ID: "al-folder", Name: "Folder Album", FolderIDs: []string{"f1"}}, + {ID: "al-embed", Name: "Embedded Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + ds := &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + MockedAlbum: albumRepo, + } + store := NewImageStore(GinkgoT().TempDir()) + deps := &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm} + conf.Server.CoverArtPriority = "cover.jpg, embedded" + + // Dangling refs (al/ra ids the repos don't know about) mirror an entity + // deleted after being enqueued; ds.Radio auto-provisions an empty mock repo. + items := []model.ArtworkQueueItem{ + {ItemKind: "al", ItemID: "al-folder"}, + {ItemKind: "al", ItemID: "al-embed"}, + {ItemKind: "al", ItemID: "al-does-not-exist"}, + {ItemKind: "ra", ItemID: "ra-does-not-exist"}, + } + + fdCount := func() int { + if runtime.GOOS != "linux" { + return -1 + } + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + return -1 + } + return len(entries) + } + + settleGoroutines := func() int { + // Background goroutines (GC workers, etc.) can take a moment to wind down; + // poll for two consecutive equal samples instead of trusting a single one. + prev := -1 + for range 100 { + runtime.GC() + n := runtime.NumGoroutine() + if n == prev { + return n + } + prev = n + time.Sleep(10 * time.Millisecond) + } + return prev + } + + baselineGoroutines := settleGoroutines() + baselineFDs := fdCount() + + var heapAt10Pct uint64 + start := time.Now() + for i := range soakCycles { + it := items[i%len(items)] + out := processItem(context.Background(), deps, it) + + // "Serve-adjacent" read-back: exercise the Phase 2 surfaces a caller would + // use after acquisition, not the old serving pipeline. + if out == outcomeFound { + ia, err := artRepo.GetItemArtwork(it.ItemKind, it.ItemID, model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred(), "cycle %d: GetItemArtwork", i) + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred(), "cycle %d: GetImage", i) + rc, err := store.Open(ia.Hash, art.Mime) + switch { + case err == nil: + _, _ = io.Copy(io.Discard, rc) + rc.Close() + case os.IsNotExist(err): + // Folder-backed art has no store file; that's expected. + default: + Expect(err).ToNot(HaveOccurred(), "cycle %d: store.Open", i) + } + } + + if i == soakCycles/10 { + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + heapAt10Pct = ms.HeapAlloc + } + } + elapsed := time.Since(start) + + finalGoroutines := settleGoroutines() + finalFDs := fdCount() + + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + + GinkgoWriter.Printf("soak: cycles=%d elapsed=%s goroutines(baseline=%d final=%d) heap(10%%-mark=%d final=%d) fds(baseline=%d final=%d)\n", + soakCycles, elapsed, baselineGoroutines, finalGoroutines, heapAt10Pct, ms.HeapAlloc, baselineFDs, finalFDs) + + Expect(finalGoroutines).To(BeNumerically("<=", baselineGoroutines), "goroutine count grew: baseline=%d final=%d", baselineGoroutines, finalGoroutines) + if heapAt10Pct > 0 { + Expect(ms.HeapAlloc).To(BeNumerically("<=", 2*heapAt10Pct), "heap did not plateau: 10%%-mark=%d final=%d (final > 2x 10%%-mark)", heapAt10Pct, ms.HeapAlloc) + } + if runtime.GOOS == "linux" && baselineFDs >= 0 { + Expect(finalFDs).To(BeNumerically("<=", baselineFDs), "fd count grew: baseline=%d final=%d", baselineFDs, finalFDs) + } + }) +}) diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go new file mode 100644 index 000000000..e1b0c985d --- /dev/null +++ b/core/artwork/worker_test.go @@ -0,0 +1,362 @@ +package artwork + +import ( + "context" + "errors" + "io" + "net/url" + "os" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/goleak" +) + +// reenqueueOnDequeue simulates a concurrent scan Enqueue between DequeueBatch and the +// worker's delete by bumping retry_at, so a DeleteIfUnchanged on the dequeued value no-ops. +type reenqueueOnDequeue struct { + *tests.MockArtworkQueueRepo + done bool +} + +func (r *reenqueueOnDequeue) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + items, err := r.MockArtworkQueueRepo.DequeueBatch(n) + if !r.done && len(items) > 0 { + r.done = true + for k, it := range r.Data { + if it.ItemKind == items[0].ItemKind && it.ItemID == items[0].ItemID { + it.RetryAt = items[0].RetryAt.Add(time.Minute) + r.Data[k] = it + } + } + } + return items, err +} + +func findQueued(q *tests.MockArtworkQueueRepo, kind, id string) *model.ArtworkQueueItem { + for _, it := range q.Data { + if it.ItemKind == kind && it.ItemID == id { + return &it + } + } + return nil +} + +var _ = Describe("Worker", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + folderRepo *fakeFolderRepo + libRepo *tests.MockLibraryRepo + ffm *tests.MockFFmpeg + prov *fakeExternalProvider + store *ImageStore + artRepo *tests.MockArtworkRepo + queueRepo *tests.MockArtworkQueueRepo + repoRoot string + w *Worker + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + folderRepo = &fakeFolderRepo{} + libRepo = &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + ffm = tests.NewMockFFmpeg("") + prov = &fakeExternalProvider{} + artRepo = tests.CreateMockArtworkRepo() + queueRepo = tests.CreateMockArtworkQueueRepo() + ds = &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + MockedArtworkQueue: queueRepo, + } + ds.MockedAlbum = tests.CreateMockAlbumRepo() + store = NewImageStore(GinkgoT().TempDir()) + conf.Server.CoverArtPriority = "cover.jpg, embedded" + conf.Server.ArtworkExternalMaxRPS = 1000 // keep the limiter out of the way of behavior tests + w = NewWorker(ds, store, prov, ffm) + }) + + Describe("drain", func() { + It("processes a seeded queue item and removes it from the queue", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}, + }) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ + ItemKind: "al", ItemID: "al1", Priority: model.ArtworkPriorityScan, + })).To(Succeed()) + + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + ia, err := artRepo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder")) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero(), "a found item must be deleted from the queue") + }) + + It("reschedules a failed item via MarkFailed with a backed-off retry_at", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al4", Name: "Album"}}) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})).To(Succeed()) + + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + it := findQueued(queueRepo, "al", "al4") + Expect(it).ToNot(BeNil()) + Expect(it.Attempts).To(Equal(1)) + Expect(it.RetryAt).To(BeTemporally(">", time.Now())) + + _, err = artRepo.GetItemArtwork("al", "al4", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound), "a timeout must never settle on absent") + }) + + It("reschedules a found-stale item via MarkFailed while keeping its served state", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "alstale", Name: "Album", FolderIDs: []string{"f1"}}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"})).To(Succeed()) + + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + it := findQueued(queueRepo, "al", "alstale") + Expect(it).ToNot(BeNil(), "a found-stale row must survive for a higher-priority retry") + Expect(it.Attempts).To(Equal(1)) + Expect(it.RetryAt).To(BeTemporally(">", time.Now())) + + ia, err := artRepo.GetItemArtwork("al", "alstale", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder"), "the fallback art is served meanwhile") + }) + + It("keeps a row re-enqueued between dequeue and delete", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al7", Name: "Album", FolderIDs: []string{"f1"}}, + }) + racing := &reenqueueOnDequeue{MockArtworkQueueRepo: queueRepo} + ds.MockedArtworkQueue = racing + w = NewWorker(ds, store, prov, ffm) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ + ItemKind: "al", ItemID: "al7", Priority: model.ArtworkPriorityScan, + })).To(Succeed()) + + n, err := w.drain(ctx, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + // The concurrent re-enqueue changed retry_at, so the found-path delete was a no-op. + Expect(findQueued(queueRepo, "al", "al7")).ToNot(BeNil()) + ia, err := artRepo.GetItemArtwork("al", "al7", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder")) + }) + + It("keeps a fresh re-enqueue ahead of a stale failure backoff", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al8", Name: "Album"}}) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + racing := &reenqueueOnDequeue{MockArtworkQueueRepo: queueRepo} + ds.MockedArtworkQueue = racing + w = NewWorker(ds, store, prov, ffm) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al8"})).To(Succeed()) + dequeued := findQueued(queueRepo, "al", "al8").RetryAt + + n, err := w.drain(ctx, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + // The concurrent re-enqueue reset retry_at; the failure path must not stomp it + // with stale backoff nor bump attempts, so the row stays immediately eligible. + it := findQueued(queueRepo, "al", "al8") + Expect(it).ToNot(BeNil()) + Expect(it.Attempts).To(BeZero()) + Expect(it.RetryAt).To(BeTemporally("==", dequeued.Add(time.Minute))) + }) + + It("resolves a private playlist under an admin context instead of failing forever", func() { + ds.MockedUser = adminUserRepo() + vds := &visibilityPlaylistDS{ + MockDataStore: ds, + private: model.Playlist{ID: "plPriv", OwnerID: "admin"}, + tracks: &tests.MockPlaylistTrackRepo{}, + } + w = NewWorker(vds, store, prov, ffm) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plPriv"})).To(Succeed()) + + n, err := w.drain(ctx, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + // Resolved as absent (no art) and removed — not stuck failing on ErrNotFound forever. + Expect(findQueued(queueRepo, "pl", "plPriv")).To(BeNil()) + ia, err := artRepo.GetItemArtwork("pl", "plPriv", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).To(BeEmpty()) + }) + + It("returns zero when the queue is empty", func() { + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(BeZero()) + }) + }) + + Describe("Bump", func() { + It("enqueues at Bump priority and wakes the loop", func() { + w.Bump("al", "al9") + it := findQueued(queueRepo, "al", "al9") + Expect(it).ToNot(BeNil()) + Expect(it.Priority).To(Equal(model.ArtworkPriorityBump)) + }) + }) + + Describe("gate/breaker", func() { + It("opens after 5 consecutive external errors and short-circuits the step", func() { + var calls int + failing := func() (io.ReadCloser, string, error) { + calls++ + return nil, "", errors.New("boom") + } + for range 5 { + _, _, err := w.gate(failing) + Expect(err).To(HaveOccurred()) + } + Expect(calls).To(Equal(5)) + + _, _, err := w.gate(failing) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(5), "an open breaker must not call the external step") + }) + + It("resets the failure count on a successful call", func() { + failing := func() (io.ReadCloser, string, error) { return nil, "", errors.New("boom") } + ok := func() (io.ReadCloser, string, error) { return io.NopCloser(nil), "p", nil } + for range 4 { + _, _, _ = w.gate(failing) + } + _, _, err := w.gate(ok) + Expect(err).ToNot(HaveOccurred()) + + var calls int + counting := func() (io.ReadCloser, string, error) { + calls++ + return nil, "", errors.New("boom") + } + for range 5 { + _, _, _ = w.gate(counting) + } + Expect(calls).To(Equal(5), "the breaker should have re-closed after the success") + }) + }) + + Describe("RunPrune", func() { + It("runs a prune under the worker mutex", func() { + Expect(w.RunPrune(ctx)).To(Succeed()) + }) + }) + + Describe("Run", func() { + It("exits cleanly when the context is cancelled", func() { + runCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- w.Run(runCtx) }() + + cancel() + Eventually(done, time.Second).Should(Receive(BeNil())) + }) + + It("does not leak goroutines after Run exits", func() { + DeferCleanup(configtest.SetupConfig()) + + ignore := goleak.IgnoreCurrent() + DeferCleanup(func() { goleak.VerifyNone(GinkgoT(), ignore) }) + + localDS := &tests.MockDataStore{MockedArtworkQueue: tests.CreateMockArtworkQueueRepo()} + lw := NewWorker(localDS, NewImageStore(GinkgoT().TempDir()), &fakeExternalProvider{}, tests.NewMockFFmpeg("")) + + runCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- lw.Run(runCtx) }() + + time.Sleep(20 * time.Millisecond) // let the loop settle on the idle select + cancel() + Eventually(done, 2*time.Second).Should(Receive(BeNil())) + }) + }) +}) + +var _ = Describe("backoff", func() { + It("returns the expected schedule with no jitter", func() { + for _, c := range []struct { + attempts int + want time.Duration + }{ + {0, 5 * time.Minute}, + {1, 20 * time.Minute}, + {2, 80 * time.Minute}, + {3, 320 * time.Minute}, + {4, 1280 * time.Minute}, + {5, 48 * time.Hour}, + {6, 48 * time.Hour}, + } { + Expect(backoffFor(c.attempts, 0)).To(Equal(c.want), "attempt %d", c.attempts) + } + }) + + It("applies jitter proportionally", func() { + base := backoffFor(2, 0) + Expect(backoffFor(2, 0.2)).To(Equal(time.Duration(float64(base) * 1.2))) + Expect(backoffFor(2, -0.2)).To(Equal(time.Duration(float64(base) * 0.8))) + }) + + It("keeps random jitter within +/-20%", func() { + lo := time.Duration(float64(320*time.Minute) * 0.8) + hi := time.Duration(float64(320*time.Minute) * 1.2) + for range 200 { + d := backoff(3) + Expect(d).To(BeNumerically(">=", lo)) + Expect(d).To(BeNumerically("<=", hi)) + } + }) +}) diff --git a/core/artwork/worker_timing_test.go b/core/artwork/worker_timing_test.go new file mode 100644 index 000000000..4bb14df3a --- /dev/null +++ b/core/artwork/worker_timing_test.go @@ -0,0 +1,39 @@ +package artwork + +import ( + "errors" + "testing" + "testing/synctest" + "time" + + . "github.com/onsi/gomega" +) + +// Drives the real breaker state machine with the fake clock. Plain test: testing/synctest +// needs a *testing.T, which Ginkgo doesn't give. +func TestArtworkBreakerHalfOpen(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + g := NewWithT(t) + b := newBreaker() + + for range breakerThreshold { + b.record(errors.New("boom")) + } + g.Expect(b.allow()).To(BeFalse(), "breaker opens after consecutive errors") + + time.Sleep(breakerProbeAfter - time.Nanosecond) + g.Expect(b.allow()).To(BeFalse(), "still open before the probe interval") + + time.Sleep(time.Nanosecond) + g.Expect(b.allow()).To(BeTrue(), "half-open: one probe is granted") + g.Expect(b.allow()).To(BeFalse(), "only a single probe per interval") + + b.record(errors.New("boom")) // probe fails -> stay open + time.Sleep(breakerProbeAfter) + g.Expect(b.allow()).To(BeTrue(), "another probe after the next interval") + + b.record(nil) // probe succeeds -> close + g.Expect(b.allow()).To(BeTrue(), "closed breaker admits freely") + g.Expect(b.allow()).To(BeTrue()) + }) +} diff --git a/core/external/provider.go b/core/external/provider.go index 459e8a205..42da7eb45 100644 --- a/core/external/provider.go +++ b/core/external/provider.go @@ -36,6 +36,8 @@ type Provider interface { SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error) TopSongs(ctx context.Context, artist string, count int) (model.MediaFiles, error) ArtistImage(ctx context.Context, id string) (*url.URL, error) + // ArtistImageResult is like ArtistImage but reports a transient agent failure as a real error, not ErrNotFound. + ArtistImageResult(ctx context.Context, id string) (*url.URL, error) AlbumImage(ctx context.Context, id string) (*url.URL, error) } @@ -258,7 +260,7 @@ func (e *provider) populateArtistInfo(ctx context.Context, artist auxArtist) (au // Call all registered agents and collect information g := errgroup.Group{} g.SetLimit(2) - g.Go(func() error { e.callGetImage(ctx, e.ag, &artist); return nil }) + g.Go(func() error { _ = e.callGetImage(ctx, e.ag, &artist); return nil }) g.Go(func() error { e.callGetBiography(ctx, e.ag, &artist); return nil }) g.Go(func() error { e.callGetURL(ctx, e.ag, &artist); return nil }) g.Go(func() error { e.callGetSimilarArtists(ctx, e.ag, &artist, maxSimilarArtists, true); return nil }) @@ -371,18 +373,35 @@ func (e *provider) similarSongsFallback(ctx context.Context, id string, count in } func (e *provider) ArtistImage(ctx context.Context, id string) (*url.URL, error) { + u, _, err := e.artistImage(ctx, id) + return u, err +} + +// ArtistImageResult is like ArtistImage but surfaces a transient agent failure as the +// real error, so an agent outage is not mistaken for a definitive no-image (ErrNotFound). +func (e *provider) ArtistImageResult(ctx context.Context, id string) (*url.URL, error) { + u, agentErr, err := e.artistImage(ctx, id) + if agentErr != nil && errors.Is(err, model.ErrNotFound) { + return nil, agentErr + } + return u, err +} + +// artistImage returns the agent error (agentErr) separately from the caller-facing err, +// so ArtistImageResult can tell "agent errored" apart from "definitively no image". +func (e *provider) artistImage(ctx context.Context, id string) (u *url.URL, agentErr error, err error) { artist, err := e.getArtist(ctx, id) if err != nil { - return nil, err + return nil, nil, err } imageUrl := artist.ArtistImageUrl() if imageUrl == "" { // No cached URL — must fetch from external source synchronously - e.callGetImage(ctx, e.ag, &artist) + agentErr = e.callGetImage(ctx, e.ag, &artist) if utils.IsCtxDone(ctx) { log.Warn(ctx, "ArtistImage call canceled", ctx.Err()) - return nil, ctx.Err() + return nil, agentErr, ctx.Err() } imageUrl = artist.ArtistImageUrl() } else { @@ -396,9 +415,10 @@ func (e *provider) ArtistImage(ctx context.Context, id string) (*url.URL, error) } if imageUrl == "" { - return nil, model.ErrNotFound + return nil, agentErr, model.ErrNotFound } - return url.Parse(imageUrl) + u, err = url.Parse(imageUrl) + return u, agentErr, err } func (e *provider) AlbumImage(ctx context.Context, id string) (*url.URL, error) { @@ -519,10 +539,15 @@ func (e *provider) callGetBiography(ctx context.Context, agent agents.ArtistBiog artist.Biography = strings.ReplaceAll(bio, " images[j].Size }) @@ -535,6 +560,7 @@ func (e *provider) callGetImage(ctx context.Context, agent agents.ArtistImageRet if len(images) >= 3 { artist.SmallImageUrl = images[2].URL } + return nil } func (e *provider) callGetSimilarArtists(ctx context.Context, agent agents.ArtistSimilarRetriever, artist *auxArtist, diff --git a/core/external/provider_artistimage_test.go b/core/external/provider_artistimage_test.go index 79612d651..621aac574 100644 --- a/core/external/provider_artistimage_test.go +++ b/core/external/provider_artistimage_test.go @@ -330,6 +330,49 @@ var _ = Describe("Provider - ArtistImage", func() { Expect(logBuf.String()).To(ContainSubstring("Artist image info expired, enqueuing background refresh")) }) + Describe("ArtistImageResult", func() { + It("returns the real agent error on a transient failure, not ErrNotFound", func() { + agentErr := errors.New("agent timed out") + mockImageAgent.Mock = mock.Mock{} + mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return(nil, agentErr).Once() + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).To(MatchError(agentErr)) + Expect(err).ToNot(MatchError(model.ErrNotFound)) + Expect(imgURL).To(BeNil()) + }) + + It("returns ErrNotFound when the agent definitively has no image", func() { + mockImageAgent.Mock = mock.Mock{} + mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return(nil, agents.ErrNotFound).Once() + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).To(MatchError(model.ErrNotFound)) + Expect(imgURL).To(BeNil()) + }) + + It("returns ErrNotFound when the agent returns no images without error", func() { + mockImageAgent.Mock = mock.Mock{} + mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return([]agents.ExternalImage{}, nil).Once() + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).To(MatchError(model.ErrNotFound)) + Expect(imgURL).To(BeNil()) + }) + + It("returns the largest image URL on success", func() { + expectedURL, _ := url.Parse("http://example.com/large.jpg") + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).ToNot(HaveOccurred()) + Expect(imgURL).To(Equal(expectedURL)) + }) + }) + Context("Unicode handling in artist names", func() { var artistWithEnDash *model.Artist var expectedURL *url.URL diff --git a/db/migrations/20260722023032_add_artwork_tables.sql b/db/migrations/20260722023032_add_artwork_tables.sql new file mode 100644 index 000000000..bc400f90e --- /dev/null +++ b/db/migrations/20260722023032_add_artwork_tables.sql @@ -0,0 +1,42 @@ +-- +goose Up +CREATE TABLE artwork ( + hash TEXT PRIMARY KEY, + mime TEXT NOT NULL, + width INTEGER NOT NULL DEFAULT 0, + height INTEGER NOT NULL DEFAULT 0, + size_bytes INTEGER NOT NULL DEFAULT 0, + blur_hash TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE item_artwork ( + item_kind TEXT NOT NULL, + item_id TEXT NOT NULL, + image_type TEXT NOT NULL DEFAULT 'primary', + hash TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + source_path TEXT NOT NULL DEFAULT '', + ref_mtime INTEGER NOT NULL DEFAULT 0, + attempted_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY (item_kind, item_id, image_type) +) WITHOUT ROWID; +CREATE INDEX ix_item_artwork_hash ON item_artwork(hash); + +CREATE TABLE artwork_queue ( + item_kind TEXT NOT NULL, + item_id TEXT NOT NULL, + image_type TEXT NOT NULL DEFAULT 'primary', + priority INTEGER NOT NULL DEFAULT 0, + attempts INTEGER NOT NULL DEFAULT 0, + retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (item_kind, item_id, image_type) +) WITHOUT ROWID; +-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering. +CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at); + +-- +goose Down +DROP TABLE artwork_queue; +DROP TABLE item_artwork; +DROP TABLE artwork; diff --git a/go.mod b/go.mod index 5488b41e4..b72f4035c 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/tetratelabs/wazero v1.12.0 github.com/unrolled/secure v1.17.0 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 + github.com/zeebo/xxh3 v1.1.0 go.senan.xyz/taglib v0.11.1 go.uber.org/goleak v1.3.0 golang.org/x/image v0.44.0 @@ -128,7 +129,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect github.com/valyala/fastjson v1.6.10 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect diff --git a/model/album.go b/model/album.go index 55df8c63c..7c03ccf1e 100644 --- a/model/album.go +++ b/model/album.go @@ -142,6 +142,7 @@ type AlbumRepository interface { UpdateExternalInfo(*Album) error Get(id string) (*Album, error) GetAll(...QueryOptions) (Albums, error) + GetAllIDs(...QueryOptions) ([]string, error) GetCursor(...QueryOptions) (AlbumCursor, error) GetYears(libraryIDs ...int) ([]int, error) diff --git a/model/artist.go b/model/artist.go index f9c4bffd5..053f261ce 100644 --- a/model/artist.go +++ b/model/artist.go @@ -89,6 +89,7 @@ type ArtistRepository interface { UpdateExternalInfo(a *Artist) error Get(id string) (*Artist, error) GetAll(options ...QueryOptions) (Artists, error) + GetAllIDs(options ...QueryOptions) ([]string, error) GetCursor(options ...QueryOptions) (ArtistCursor, error) GetIndex(includeMissing bool, libraryIds []int, roles ...Role) (ArtistIndexes, error) diff --git a/model/artwork.go b/model/artwork.go new file mode 100644 index 000000000..8db11ce69 --- /dev/null +++ b/model/artwork.go @@ -0,0 +1,103 @@ +package model + +import "time" + +// Artwork is one unique image, identified by the XXH3-64 hash of its bytes. +type Artwork struct { + Hash string `structs:"hash"` + Mime string `structs:"mime"` + Width int `structs:"width"` + Height int `structs:"height"` + SizeBytes int64 `structs:"size_bytes"` + BlurHash string `structs:"blur_hash"` + CreatedAt time.Time `structs:"created_at"` +} + +const ImageTypePrimary = "primary" + +// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent. +type ItemArtwork struct { + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Hash string `structs:"hash"` + Source string `structs:"source"` + // SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise. + SourcePath string `structs:"source_path"` + // RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath. + RefMtime int64 `structs:"ref_mtime"` + // attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork; + // raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL. + AttemptedAt time.Time `structs:"attempted_at"` + UpdatedAt time.Time `structs:"updated_at"` +} + +// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork). +type ItemArtworkInfo struct { + ItemID string + Hash string + BlurHash string +} + +// Absent reports a known-absent artwork state (resolved, no image). +func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" } + +type ArtworkQueueItem struct { + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Priority int `structs:"priority"` + Attempts int `structs:"attempts"` + RetryAt time.Time `structs:"retry_at"` + EnqueuedAt time.Time `structs:"enqueued_at"` +} + +// Queue priorities: higher drains first. +const ( + ArtworkPriorityRecheck = 0 + ArtworkPriorityBackfill = 10 + ArtworkPriorityScan = 50 + ArtworkPriorityBump = 100 +) + +type ArtworkRepository interface { + // Image identity (artwork table) + GetImage(hash string) (*Artwork, error) + PutImage(a *Artwork) error + GetImages(hashes []string) (map[string]Artwork, error) + // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. + GetOrphanHashes(createdBefore time.Time) ([]string, error) + // DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check). + DeleteOrphans(createdBefore time.Time, hashes []string) error + // Per-item state (item_artwork table) + GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error) + PutItemArtwork(ia *ItemArtwork) error + DeleteForItem(kind, id string) error + // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. + GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) + // GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks. + GetAllMimes() (map[string]string, error) + // PurgeDanglingItemArtwork removes state rows whose entity no longer exists. + PurgeDanglingItemArtwork() (int64, error) +} + +type ArtworkQueueRepository interface { + // Enqueue upserts; an existing row keeps the higher of the two priorities. + Enqueue(items ...ArtworkQueueItem) error + // DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc. + DequeueBatch(n int) ([]ArtworkQueueItem, error) + // MarkFailed increments attempts and pushes retry_at into the future. + MarkFailed(kind, id, imageType string, retryAt time.Time) error + // MarkFailedIfUnchanged applies the failure backoff only while retry_at still matches + // seenRetryAt; a concurrent re-enqueue (which resets retry_at) keeps its fresh eligibility. + MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error + Delete(kind, id, imageType string) error + // DeleteIfUnchanged deletes the row only if its retry_at still matches retryAt, so a + // concurrent re-enqueue (which resets retry_at) survives instead of being erased. + DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error + Count() (int64, error) + // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. + EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) + // PurgeDangling removes queue rows whose entity no longer exists. + PurgeDangling() (int64, error) +} diff --git a/model/datastore.go b/model/datastore.go index 94c3c3622..273ca714b 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -40,6 +40,8 @@ type DataStore interface { ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository Scrobble(ctx context.Context) ScrobbleRepository Plugin(ctx context.Context) PluginRepository + Artwork(ctx context.Context) ArtworkRepository + ArtworkQueue(ctx context.Context) ArtworkQueueRepository Resource(ctx context.Context, model any) ResourceRepository diff --git a/model/playlist.go b/model/playlist.go index 185f6f942..b5726e4ff 100644 --- a/model/playlist.go +++ b/model/playlist.go @@ -143,6 +143,7 @@ type PlaylistRepository interface { Get(id string) (*Playlist, error) GetWithTracks(id string, refreshSmartPlaylist, includeMissing bool) (*Playlist, error) GetAll(options ...QueryOptions) (Playlists, error) + GetAllIDs(options ...QueryOptions) ([]string, error) GetCursor(options ...QueryOptions) (PlaylistCursor, error) FindByPath(path string) (*Playlist, error) Delete(id string) error diff --git a/model/radio.go b/model/radio.go index 86f27c24c..7f654b665 100644 --- a/model/radio.go +++ b/model/radio.go @@ -32,5 +32,6 @@ type RadioRepository interface { Delete(id string) error Get(id string) (*Radio, error) GetAll(options ...QueryOptions) (Radios, error) + GetAllIDs(options ...QueryOptions) ([]string, error) Put(u *Radio, colsToUpdate ...string) error } diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 2bb541003..e7b155e6f 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -254,6 +254,15 @@ func (r *albumRepository) GetAll(options ...model.QueryOptions) (model.Albums, e return res.toModels(), nil } +// GetAllIDs returns just the album IDs for the same row set as GetAll, skipping the +// heavy column projection and JSON post-processing. Used by bulk enumeration (artwork backfill). +func (r *albumRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sq := r.applyLibraryFilter(r.newSelect(options...).Columns("album.id")) + ids := []string{} + err := r.queryAllSlice(sq, &ids) + return ids, err +} + func (r *albumRepository) GetCursor(options ...model.QueryOptions) (model.AlbumCursor, error) { sq := r.selectAlbum(options...) cursor, err := queryWithStableResults[dbAlbum](r.sqlRepository, sq) diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 465617b10..ef18cf426 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -84,6 +84,21 @@ var _ = Describe("AlbumRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := albumRepo.GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, a := range want { + wantIDs = append(wantIDs, a.ID) + } + ids, err := albumRepo.GetAllIDs() + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("GetAll", func() { var GetAll = func(opts ...model.QueryOptions) (model.Albums, error) { albums, err := albumRepo.GetAll(opts...) diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index b542dedb4..f26ade8ba 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -264,6 +264,15 @@ func (r *artistRepository) GetAll(options ...model.QueryOptions) (model.Artists, return res, err } +// GetAllIDs returns just the artist IDs for the same row set as GetAll, skipping the +// heavy stats/annotation columns and JSON post-processing. Used by bulk enumeration (artwork backfill). +func (r *artistRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sq := r.applyLibraryFilterToArtistQuery(r.newSelect(options...).Columns("artist.id")).GroupBy("artist.id") + ids := []string{} + err := r.queryAllSlice(sq, &ids) + return ids, err +} + func (r *artistRepository) GetCursor(options ...model.QueryOptions) (model.ArtistCursor, error) { sel := r.selectArtist(options...) cursor, err := queryWithStableResults[dbArtist](r.sqlRepository, sel) diff --git a/persistence/artist_repository_test.go b/persistence/artist_repository_test.go index dc11ede36..43a7f4928 100644 --- a/persistence/artist_repository_test.go +++ b/persistence/artist_repository_test.go @@ -284,6 +284,21 @@ var _ = Describe("ArtistRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := repo.GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, a := range want { + wantIDs = append(wantIDs, a.ID) + } + ids, err := repo.GetAllIDs() + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("Basic Operations", func() { Describe("Count", func() { It("returns the number of artists in the DB", func() { diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go new file mode 100644 index 000000000..59e130204 --- /dev/null +++ b/persistence/artwork_queue_repository.go @@ -0,0 +1,112 @@ +package persistence + +import ( + "context" + "slices" + "time" + + . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars). +const enqueueChunkSize = 100 + +type artworkQueueRepository struct { + sqlRepository +} + +func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository { + r := &artworkQueueRepository{} + r.ctx = ctx + r.db = db + r.tableName = "artwork_queue" + return r +} + +func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error { + now := time.Now() + for chunk := range slices.Chunk(items, enqueueChunkSize) { + ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at") + for _, it := range chunk { + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary + } + ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now) + } + ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`) + if _, err := r.executeSQL(ins); err != nil { + return err + } + } + return nil +} + +func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + sel := Select("*").From(r.tableName). + Where(LtOrEq{"retry_at": time.Now()}). + OrderBy("priority DESC", "enqueued_at ASC"). + Limit(uint64(n)) + var res []model.ArtworkQueueItem + err := r.queryAll(sel, &res) + return res, err +} + +func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error { + upd := Update(r.tableName). + Set("attempts", Expr("attempts + 1")). + Set("retry_at", retryAt). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + c, err := r.executeSQL(upd) + if err == nil && c == 0 { + return model.ErrNotFound + } + return err +} + +// MarkFailedIfUnchanged applies the backoff only while retry_at still equals seenRetryAt; +// a concurrent Enqueue resets retry_at, so its fresh eligibility survives untouched. +func (r *artworkQueueRepository) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error { + upd := Update(r.tableName). + Set("attempts", Expr("attempts + 1")). + Set("retry_at", retryAt). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": seenRetryAt}) + _, err := r.executeSQL(upd) + return err +} + +func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { + return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) +} + +// DeleteIfUnchanged deletes the row only while its retry_at still equals the dequeued +// value; a concurrent Enqueue resets retry_at, so the row survives to be re-resolved. +func (r *artworkQueueRepository) DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error { + return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": retryAt}) +} + +// PurgeDangling removes queue rows whose entity no longer exists, per kind. +func (r *artworkQueueRepository) PurgeDangling() (int64, error) { + return purgeDangling(r.executeSQL, r.tableName) +} + +func (r *artworkQueueRepository) Count() (int64, error) { + var res struct{ Count int64 } + err := r.queryOne(Select("count(*) as count").From(r.tableName), &res) + return res.Count, err +} + +func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + now := time.Now() + // DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items. + ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) + SELECT item_kind, item_id, image_type, ?, 0, ?, ? + FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ? + ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, + model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) + return r.executeSQL(ins) +} + +var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil) diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go new file mode 100644 index 000000000..0a6ddefdc --- /dev/null +++ b/persistence/artwork_queue_repository_test.go @@ -0,0 +1,160 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtworkQueueRepository", func() { + var repo model.ArtworkQueueRepository + + item := func(kind, id string, prio int) model.ArtworkQueueItem { + return model.ArtworkQueueItem{ItemKind: kind, ItemID: id, + ImageType: model.ImageTypePrimary, Priority: prio} + } + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + }) + + It("enqueues and dequeues by priority then FIFO", func() { + Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed()) + Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed()) + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got[0].ItemID).To(Equal("high")) + }) + + It("keeps the higher priority on duplicate enqueue", func() { + Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed()) + Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed()) + got, _ := repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump)) + }) + + It("hides failed items until retry_at", func() { + Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed()) + Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed()) + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeEmpty()) + + Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Attempts).To(Equal(2)) + }) + + It("MarkFailedIfUnchanged applies backoff only while retry_at is unchanged", func() { + Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed()) + // Anchor retry_at in the past (attempts -> 1) so it can never collide with the re-enqueue's now. + Expect(repo.MarkFailed("al", "m1", model.ImageTypePrimary, time.Now().Add(-time.Hour))).To(Succeed()) + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(1)) + original := got[0].RetryAt + + // A concurrent scan re-enqueues, resetting retry_at to now. + Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed()) + + // Failing with the stale retry_at is a no-op: the re-enqueued row keeps its fresh state. + future := time.Now().Add(48 * time.Hour) + Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, original, future)).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1), "the fresh re-enqueue stays immediately eligible") + Expect(got[0].Attempts).To(Equal(1), "the stale failure must not bump attempts") + current := got[0].RetryAt + + // Failing with the current retry_at applies the backoff and bumps attempts. + Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, current, future)).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(BeEmpty(), "backed-off row is hidden until the future retry_at") + all, _ := repo.Count() + Expect(all).To(Equal(int64(1))) + }) + + It("deletes on completion and counts", func() { + Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed()) + n, _ := repo.Count() + Expect(n).To(Equal(int64(1))) + Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed()) + n, _ = repo.Count() + Expect(n).To(BeZero()) + }) + + It("DeleteIfUnchanged deletes only while retry_at is unchanged", func() { + Expect(repo.Enqueue(item("al", "d1", model.ArtworkPriorityScan))).To(Succeed()) + // Anchor retry_at in the past so it can never collide with the re-enqueue's now. + Expect(repo.MarkFailed("al", "d1", model.ImageTypePrimary, time.Now().Add(-time.Hour))).To(Succeed()) + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(1)) + original := got[0].RetryAt + + // A concurrent scan re-enqueues, resetting retry_at to now. + Expect(repo.Enqueue(item("al", "d1", model.ArtworkPriorityScan))).To(Succeed()) + + // Deleting with the stale retry_at is a no-op: the re-enqueued row survives. + Expect(repo.DeleteIfUnchanged("al", "d1", model.ImageTypePrimary, original)).To(Succeed()) + n, _ := repo.Count() + Expect(n).To(Equal(int64(1))) + + // Deleting with the current retry_at removes it. + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(repo.DeleteIfUnchanged("al", "d1", model.ImageTypePrimary, got[0].RetryAt)).To(Succeed()) + n, _ = repo.Count() + Expect(n).To(BeZero()) + }) + + It("purges queue rows whose entity no longer exists, per kind", func() { + Expect(repo.Enqueue( + item("al", albumSgtPeppers.ID, model.ArtworkPriorityScan), + item("al", "no-such-album", model.ArtworkPriorityScan), + item("ar", artistKraftwerk.ID, model.ArtworkPriorityScan), + item("ar", "no-such-artist", model.ArtworkPriorityScan), + item("pl", plsBest.ID, model.ArtworkPriorityScan), + item("pl", "no-such-playlist", model.ArtworkPriorityScan), + item("ra", radioWithHomePage.ID, model.ArtworkPriorityScan), + item("ra", "no-such-radio", model.ArtworkPriorityScan), + )).To(Succeed()) + + purged, err := repo.PurgeDangling() + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(Equal(int64(4))) + + got, _ := repo.DequeueBatch(100) + ids := make([]string, 0, len(got)) + for _, it := range got { + ids = append(ids, it.ItemID) + } + Expect(ids).To(ConsistOf(albumSgtPeppers.ID, artistKraftwerk.ID, plsBest.ID, radioWithHomePage.ID)) + }) + + It("enqueues stale absent states for recheck", func() { + awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) + old := time.Now().Add(-48 * time.Hour) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) + + n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + items, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(HaveLen(1)) + Expect(items[0].ItemID).To(Equal("stale1")) + Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) + }) +}) diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go new file mode 100644 index 000000000..9ac32b42e --- /dev/null +++ b/persistence/artwork_repository.go @@ -0,0 +1,208 @@ +package persistence + +import ( + "context" + "slices" + "time" + + . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +const ( + itemArtworkTable = "item_artwork" + artworkBatchSize = 200 +) + +type itemArtworkSQL struct { + sqlRepository +} + +type artworkRepository struct { + sqlRepository + items itemArtworkSQL +} + +func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository { + r := &artworkRepository{} + r.ctx = ctx + r.db = db + r.tableName = "artwork" + r.items.ctx = ctx + r.items.db = db + r.items.tableName = itemArtworkTable + return r +} + +func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) { + sel := Select("*").From(r.tableName).Where(Eq{"hash": hash}) + var res model.Artwork + if err := r.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil +} + +func (r *artworkRepository) PutImage(a *model.Artwork) error { + // created_at is the last-acquisition-write time the prune grace window keys on. + a.CreatedAt = time.Now() + values, err := toSQLArgs(*a) + if err != nil { + return err + } + // created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window. + ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, + height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`) + _, err = r.executeSQL(ins) + return err +} + +func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) { + res := map[string]model.Artwork{} + for chunk := range slices.Chunk(hashes, artworkBatchSize) { + sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk}) + var all []model.Artwork + if err := r.queryAll(sel, &all); err != nil { + return nil, err + } + for _, a := range all { + res[a.Hash] = a + } + } + return res, nil +} + +func (r *artworkRepository) GetAllMimes() (map[string]string, error) { + sel := Select("hash", "mime").From(r.tableName) + var rows []struct { + Hash string + Mime string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + res := make(map[string]string, len(rows)) + for _, row := range rows { + res[row.Hash] = row.Mime + } + return res, nil +} + +func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) { + sel := Select("hash").From(r.tableName). + Where(And{ + Lt{"created_at": createdBefore}, + Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"), + }) + var hashes []string + err := r.queryAllSlice(sel, &hashes) + return hashes, err +} + +func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error { + for chunk := range slices.Chunk(hashes, artworkBatchSize) { + del := Delete(r.tableName).Where(And{ + Eq{"hash": chunk}, + Lt{"created_at": createdBefore}, + Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"), + }) + if _, err := r.executeSQL(del); err != nil { + return err + } + } + return nil +} + +// danglingItemArtworkKinds maps item_kind prefixes to the table that owns the entity. +var danglingItemArtworkKinds = map[string]string{ + "al": "album", + "ar": "artist", + "pl": "playlist", + "ra": "radio", +} + +// purgeDangling deletes rows in table whose owning entity is gone, one statement per kind. +func purgeDangling(execute func(Sqlizer) (int64, error), table string) (int64, error) { + var total int64 + for kind, entityTable := range danglingItemArtworkKinds { + del := Delete(table).Where(And{ + Eq{"item_kind": kind}, + Expr("item_id NOT IN (SELECT id FROM " + entityTable + ")"), + }) + c, err := execute(del) + if err != nil { + return total, err + } + total += c + } + return total, nil +} + +func (r *artworkRepository) PurgeDanglingItemArtwork() (int64, error) { + return purgeDangling(r.items.executeSQL, itemArtworkTable) +} + +func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { + sel := Select("*").From(itemArtworkTable). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + var res model.ItemArtwork + if err := r.items.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil +} + +func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error { + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } + ia.UpdatedAt = time.Now() + // PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now. + if ia.AttemptedAt.IsZero() { + ia.AttemptedAt = ia.UpdatedAt + } + values, err := toSQLArgs(*ia) + if err != nil { + return err + } + ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime, + attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) + _, err = r.items.executeSQL(ins) + return err +} + +func (r *artworkRepository) DeleteForItem(kind, id string) error { + return r.items.delete(Eq{"item_kind": kind, "item_id": id}) +} + +func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + res := map[string]model.ItemArtworkInfo{} + for chunk := range slices.Chunk(ids, artworkBatchSize) { + sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). + From(itemArtworkTable + " ia"). + LeftJoin("artwork a ON a.hash = ia.hash"). + Where(And{ + Eq{"ia.item_kind": kind}, + Eq{"ia.image_type": model.ImageTypePrimary}, + Eq{"ia.item_id": chunk}, + }) + var rows []struct { + ItemID string + Hash string + BlurHash string + } + if err := r.items.queryAll(sel, &rows); err != nil { + return nil, err + } + for _, row := range rows { + res[row.ItemID] = model.ItemArtworkInfo{ + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, + } + } + } + return res, nil +} + +var _ model.ArtworkRepository = (*artworkRepository)(nil) diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go new file mode 100644 index 000000000..ae6752def --- /dev/null +++ b/persistence/artwork_repository_test.go @@ -0,0 +1,233 @@ +package persistence + +import ( + "context" + "fmt" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pocketbase/dbx" +) + +// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state. +func clearArtworkTables() { + db := GetDBXBuilder() + for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} { + _, err := db.NewQuery("DELETE FROM " + t).Execute() + Expect(err).ToNot(HaveOccurred()) + } +} + +var _ = Describe("ArtworkRepository", func() { + var repo model.ArtworkRepository + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkRepository(context.Background(), GetDBXBuilder()) + }) + + Context("image identity", func() { + It("stores and retrieves an artwork by hash", func() { + a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"} + Expect(repo.PutImage(a)).To(Succeed()) + + got, err := repo.GetImage("abc123") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Mime).To(Equal("image/jpeg")) + Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) + Expect(got.CreatedAt).ToNot(BeZero()) + }) + + It("is idempotent on Put (upsert by hash)", func() { + a := &model.Artwork{Hash: "dup1", Mime: "image/png"} + Expect(repo.PutImage(a)).To(Succeed()) + a.BlurHash = "XYZ" + Expect(repo.PutImage(a)).To(Succeed()) + got, _ := repo.GetImage("dup1") + Expect(got.BlurHash).To(Equal("XYZ")) + }) + + It("refreshes created_at when reacquiring an existing hash", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed()) + _, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'"). + Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute() + Expect(err).ToNot(HaveOccurred()) + + Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed()) + + got, err := repo.GetImage("reacq") + Expect(err).ToNot(HaveOccurred()) + Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) + }) + + It("returns ErrNotFound for a missing hash", func() { + _, err := repo.GetImage("nope") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("fetches a batch", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) + got, err := repo.GetImages([]string{"b1", "b2", "missing"}) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got["b2"].Mime).To(Equal("image/png")) + }) + + It("returns every stored hash with its current mime", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed()) + mimes, err := repo.GetAllMimes() + Expect(err).ToNot(HaveOccurred()) + Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg")) + Expect(mimes).To(HaveKeyWithValue("all2", "image/png")) + }) + + It("finds orphans older than cutoff, honoring item_artwork references", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) + + orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(ContainElement("orph1")) + Expect(orphans).ToNot(ContainElement("ref1")) + + orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(BeEmpty()) + }) + + It("deletes only unreferenced hashes older than the cutoff", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", + ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed()) + + Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed()) + + _, err := repo.GetImage("d1") + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = repo.GetImage("dref") + Expect(err).ToNot(HaveOccurred()) + }) + + It("spares an unreferenced hash younger than the cutoff", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed()) + _, err := repo.GetImage("young") + Expect(err).ToNot(HaveOccurred()) + }) + + It("fetches a batch larger than the SQL variable limit", func() { + hashes := make([]string, 0, 250) + for i := range 250 { + h := fmt.Sprintf("big%03d", i) + Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + hashes = append(hashes, h) + } + hashes = append(hashes, "absent1", "absent2") + + got, err := repo.GetImages(hashes) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(250)) + }) + }) + + Context("dangling state cleanup", func() { + It("purges item_artwork rows per kind whose entity no longer exists, summing counts", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "keepAl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "no-such-album", ImageType: model.ImageTypePrimary, Hash: "danglingAl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: artistKraftwerk.ID, ImageType: model.ImageTypePrimary, Hash: "keepAr"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "no-such-artist", ImageType: model.ImageTypePrimary, Hash: "danglingAr"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: plsBest.ID, ImageType: model.ImageTypePrimary, Hash: "keepPl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "no-such-playlist", ImageType: model.ImageTypePrimary, Hash: "danglingPl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ra", ItemID: radioWithHomePage.ID, ImageType: model.ImageTypePrimary, Hash: "keepRa"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ra", ItemID: "no-such-radio", ImageType: model.ImageTypePrimary, Hash: "danglingRa"})).To(Succeed()) + + purged, err := repo.PurgeDanglingItemArtwork() + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(Equal(int64(4))) + + for _, kept := range []model.ItemArtwork{ + {ItemKind: "al", ItemID: albumSgtPeppers.ID}, + {ItemKind: "ar", ItemID: artistKraftwerk.ID}, + {ItemKind: "pl", ItemID: plsBest.ID}, + {ItemKind: "ra", ItemID: radioWithHomePage.ID}, + } { + _, err := repo.GetItemArtwork(kept.ItemKind, kept.ItemID, model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + } + for _, gone := range []model.ItemArtwork{ + {ItemKind: "al", ItemID: "no-such-album"}, + {ItemKind: "ar", ItemID: "no-such-artist"}, + {ItemKind: "pl", ItemID: "no-such-playlist"}, + {ItemKind: "ra", ItemID: "no-such-radio"}, + } { + _, err := repo.GetItemArtwork(gone.ItemKind, gone.ItemID, model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + } + }) + }) + + Context("item state", func() { + It("upserts and reads state, including per-item provenance", func() { + ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, + Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()} + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + ia.Source = "embedded" + ia.SourcePath = "/music/a/track.mp3" + ia.RefMtime = 222 + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + + got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Source).To(Equal("embedded")) + Expect(got.SourcePath).To(Equal("/music/a/track.mp3")) + Expect(got.RefMtime).To(Equal(int64(222))) + Expect(got.UpdatedAt).ToNot(BeZero()) + }) + + It("defaults attempted_at to now when unset", func() { + before := time.Now().Add(-time.Second) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt", + ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.AttemptedAt).To(BeTemporally(">", before)) + }) + + It("represents known-absent as empty hash", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", + ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Hash).To(BeEmpty()) + }) + + It("hydrates a page in one batch, including blurhash and absence", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) + + info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(info).To(HaveLen(2)) + Expect(info["x1"].Hash).To(Equal("h9")) + Expect(info["x1"].BlurHash).To(Equal("BH9")) + Expect(info["x1"].Absent()).To(BeFalse()) + Expect(info["x2"].Absent()).To(BeTrue()) + _, unresolved := info["x3"] + Expect(unresolved).To(BeFalse()) + }) + + It("deletes all rows for an item", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) + Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) + _, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + }) +}) diff --git a/persistence/persistence.go b/persistence/persistence.go index 93f0e3e71..9d3a33cfc 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -97,6 +97,14 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository { return NewPluginRepository(ctx, s.getDBXBuilder()) } +func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository { + return NewArtworkRepository(ctx, s.getDBXBuilder()) +} + +func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { + return NewArtworkQueueRepository(ctx, s.getDBXBuilder()) +} + func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository { switch m.(type) { case model.User: diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.go index c78e0df1e..0abf53d93 100644 --- a/persistence/playlist_repository.go +++ b/persistence/playlist_repository.go @@ -189,6 +189,16 @@ func (r *playlistRepository) GetAll(options ...model.QueryOptions) (model.Playli return playlists, err } +// GetAllIDs returns just the playlist IDs for the same row set as GetAll (honoring userFilter), +// skipping the owner-name join columns and annotation. Used by bulk enumeration (artwork backfill). +func (r *playlistRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sq := r.newSelect(options...).Columns("playlist.id"). + Join("user on user.id = owner_id").Where(r.userFilter()) + ids := []string{} + err := r.queryAllSlice(sq, &ids) + return ids, err +} + func (r *playlistRepository) GetCursor(options ...model.QueryOptions) (model.PlaylistCursor, error) { // Same userFilter as GetAll: a cursor must not widen visibility beyond public/owned playlists. sel := r.selectPlaylist(options...).Where(r.userFilter()) diff --git a/persistence/playlist_repository_test.go b/persistence/playlist_repository_test.go index 00d4ff9f2..8dc86d887 100644 --- a/persistence/playlist_repository_test.go +++ b/persistence/playlist_repository_test.go @@ -37,6 +37,21 @@ var _ = Describe("PlaylistRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := repo.GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, p := range want { + wantIDs = append(wantIDs, p.ID) + } + ids, err := repo.GetAllIDs() + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("Exists", func() { It("returns true for an existing playlist", func() { Expect(repo.Exists(plsCool.ID)).To(BeTrue()) diff --git a/persistence/radio_repository.go b/persistence/radio_repository.go index a073643db..3ef900920 100644 --- a/persistence/radio_repository.go +++ b/persistence/radio_repository.go @@ -7,6 +7,7 @@ import ( . "github.com/Masterminds/squirrel" "github.com/deluan/rest" + "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/id" "github.com/pocketbase/dbx" @@ -58,6 +59,14 @@ func (r *radioRepository) GetAll(options ...model.QueryOptions) (model.Radios, e return res, err } +// GetAllIDs returns just the radio IDs. Used by bulk enumeration (artwork backfill). +func (r *radioRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sel := r.newSelect(options...).Columns("id") + ids := []string{} + err := r.queryAllSlice(sel, &ids) + return ids, err +} + func (r *radioRepository) Put(radio *model.Radio, colsToUpdate ...string) error { if !r.isPermitted() { return rest.ErrPermissionDenied @@ -72,7 +81,16 @@ func (r *radioRepository) Put(radio *model.Radio, colsToUpdate ...string) error colsToUpdate = append(colsToUpdate, "UpdatedAt") } _, err := r.put(radio.ID, radio, colsToUpdate...) - return err + if err != nil { + return err + } + // Enqueue artwork resolution for the created/updated radio. Never fails the save. + item := model.ArtworkQueueItem{ItemKind: "ra", ItemID: radio.ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan} + if err := NewArtworkQueueRepository(r.ctx, r.db).Enqueue(item); err != nil { + log.Warn(r.ctx, "could not enqueue radio artwork", "id", radio.ID, err) + } + return nil } func (r *radioRepository) Count(options ...rest.QueryOptions) (int64, error) { diff --git a/persistence/radio_repository_test.go b/persistence/radio_repository_test.go index 05628ca41..4b243c142 100644 --- a/persistence/radio_repository_test.go +++ b/persistence/radio_repository_test.go @@ -78,6 +78,21 @@ var _ = Describe("RadioRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := repo.GetAll() + Expect(err).To(BeNil()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, r := range want { + wantIDs = append(wantIDs, r.ID) + } + ids, err := repo.GetAllIDs() + Expect(err).To(BeNil()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("Put", func() { It("successfully updates item", func() { err := repo.Put(&model.Radio{ @@ -107,6 +122,27 @@ var _ = Describe("RadioRepository", func() { Expect(err).To(BeNil()) Expect(all[2].StreamUrl).To(Equal("https://example.com:4533/app")) }) + + It("enqueues artwork resolution for the saved radio", func() { + err := repo.Put(&model.Radio{ + Name: "Artwork radio", + StreamUrl: "https://example.com:4533/artwork", + }) + Expect(err).To(BeNil()) + + all, err := repo.GetAll() + Expect(err).To(BeNil()) + created := all[len(all)-1] + + queueRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + queued, err := queueRepo.DequeueBatch(1000) + Expect(err).To(BeNil()) + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "ra"), + HaveField("ItemID", created.ID), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + }) }) }) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 5e898590b..3a7265cc4 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -332,6 +332,8 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) // Collect artwork IDs to pre-cache after the transaction commits var artworkIDs []model.ArtworkID + // Collect artwork queue items for changed albums/artists, enqueued in the same transaction + var queueItems []model.ArtworkQueueItem err := p.ds.WithTx(func(tx model.DataStore) error { // Instantiate all repositories just once per folder @@ -372,6 +374,10 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) } if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists { artworkIDs = append(artworkIDs, entry.artists[i].CoverArtID()) + queueItems = append(queueItems, model.ArtworkQueueItem{ + ItemKind: "ar", ItemID: entry.artists[i].ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan, + }) } } @@ -384,6 +390,10 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) } if entry.albums[i].Name != consts.UnknownAlbum { artworkIDs = append(artworkIDs, entry.albums[i].CoverArtID()) + queueItems = append(queueItems, model.ArtworkQueueItem{ + ItemKind: "al", ItemID: entry.albums[i].ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan, + }) } } @@ -415,6 +425,13 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) return err } } + + // Enqueue artwork resolution for changed albums/artists. Never fails the scan. + if len(queueItems) > 0 { + if err := tx.ArtworkQueue(p.ctx).Enqueue(queueItems...); err != nil { + log.Warn(p.ctx, "Scanner: could not enqueue artwork resolution", "folder", entry.path, err) + } + } return nil }, "scanner: persist changes") if err != nil { diff --git a/scanner/phase_4_playlists.go b/scanner/phase_4_playlists.go index 8ba014235..15798b054 100644 --- a/scanner/phase_4_playlists.go +++ b/scanner/phase_4_playlists.go @@ -149,6 +149,11 @@ func (p *phasePlaylists) processPlaylistsInFolder(folder *model.Folder) (*model. log.Debug("Scanner: Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks), "elapsed", time.Since(started)) } p.cw.PreCache(pls.CoverArtID()) + item := model.ArtworkQueueItem{ItemKind: "pl", ItemID: pls.ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan} + if err := p.ds.ArtworkQueue(p.ctx).Enqueue(item); err != nil { + log.Warn(p.ctx, "Scanner: could not enqueue playlist artwork", "id", pls.ID, err) + } p.refreshed.Add(1) } return folder, nil diff --git a/scanner/phase_4_playlists_test.go b/scanner/phase_4_playlists_test.go index 49ffc7fb7..1adf34a63 100644 --- a/scanner/phase_4_playlists_test.go +++ b/scanner/phase_4_playlists_test.go @@ -193,6 +193,29 @@ var _ = Describe("phasePlaylists", func() { Expect(phase.refreshed.Load()).To(Equal(uint32(2))) }) + It("enqueues artwork resolution for the imported playlist", func() { + libPath := GinkgoT().TempDir() + folder := &model.Folder{LibraryPath: libPath, Path: "path/to", Name: "folder"} + _ = os.MkdirAll(folder.AbsolutePath(), 0755) + + file1 := filepath.Join(folder.AbsolutePath(), "playlist1.m3u") + _ = os.WriteFile(file1, []byte{}, 0600) + + pls.On("ImportFromFolder", mock.Anything, folder, "playlist1.m3u"). + Return(&model.Playlist{ID: "pl1"}, nil) + + _, err := phase.processPlaylistsInFolder(folder) + Expect(err).ToNot(HaveOccurred()) + + queued, err := ds.ArtworkQueue(ctx).DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "pl"), + HaveField("ItemID", "pl1"), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + }) + It("reports an error if there is an error reading files", func() { tests.SkipOnWindows("relies on Unix /etc filesystem") progress := make(chan *ProgressInfo) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index cc3720717..0a7fb07b2 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -152,6 +152,29 @@ var _ = Describe("Scanner", Ordered, func() { HaveField("SongCount", Equal(4)), )) }) + It("should enqueue artwork resolution for the scanned albums and artists", func() { + Expect(runScanner(ctx, true)).To(Succeed()) + + albums, _ := ds.Album(ctx).GetAll() + artists, _ := ds.Artist(ctx).GetAll(model.QueryOptions{Filters: squirrel.NotEq{"name": consts.UnknownArtist}}) + queued, err := ds.ArtworkQueue(ctx).DequeueBatch(1000) + Expect(err).ToNot(HaveOccurred()) + + for _, al := range albums { + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "al"), + HaveField("ItemID", al.ID), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + } + for _, ar := range artists { + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "ar"), + HaveField("ItemID", ar.ID), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + } + }) }) When("a file was changed", func() { It("should update the media_file", func() { diff --git a/server/subsonic/e2e/e2e_suite_test.go b/server/subsonic/e2e/e2e_suite_test.go index 6875b6370..98ad6d1ee 100644 --- a/server/subsonic/e2e/e2e_suite_test.go +++ b/server/subsonic/e2e/e2e_suite_test.go @@ -361,6 +361,10 @@ func (n noopProvider) ArtistImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } +func (n noopProvider) ArtistImageResult(context.Context, string) (*url.URL, error) { + return nil, model.ErrNotFound +} + func (n noopProvider) AlbumImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } diff --git a/tests/mock_album_repo.go b/tests/mock_album_repo.go index cc7d66b4d..17539255b 100644 --- a/tests/mock_album_repo.go +++ b/tests/mock_album_repo.go @@ -76,6 +76,18 @@ func (m *MockAlbumRepo) GetAll(qo ...model.QueryOptions) (model.Albums, error) { return m.All, nil } +func (m *MockAlbumRepo) GetAllIDs(qo ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(qo...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, a := range all { + ids[i] = a.ID + } + return ids, nil +} + func (m *MockAlbumRepo) GetCursor(qo ...model.QueryOptions) (model.AlbumCursor, error) { res, err := m.GetAll(qo...) if err != nil { diff --git a/tests/mock_artist_repo.go b/tests/mock_artist_repo.go index e6ea7aea4..4fdd198fe 100644 --- a/tests/mock_artist_repo.go +++ b/tests/mock_artist_repo.go @@ -113,6 +113,18 @@ func (m *MockArtistRepo) GetAll(options ...model.QueryOptions) (model.Artists, e return allArtists, nil } +func (m *MockArtistRepo) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(options...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, a := range all { + ids[i] = a.ID + } + return ids, nil +} + func (m *MockArtistRepo) GetCursor(options ...model.QueryOptions) (model.ArtistCursor, error) { res, err := m.GetAll(options...) if err != nil { diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go new file mode 100644 index 000000000..421e0dd15 --- /dev/null +++ b/tests/mock_artwork_queue_repo.go @@ -0,0 +1,169 @@ +package tests + +import ( + "sort" + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockArtworkQueueRepo struct { + model.ArtworkQueueRepository + Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType) + Err error + // ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state. + ItemArtworkSource *MockArtworkRepo + // ExistingIDs, keyed by item_kind, backs PurgeDangling; a nil per-kind map keeps that kind. + ExistingIDs map[string]map[string]bool +} + +func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo { + return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}} +} + +func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { + if m.Err != nil { + return m.Err + } + now := time.Now() + for _, it := range items { + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary + } + k := iaKey(it.ItemKind, it.ItemID, it.ImageType) + // Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller. + if prev, ok := m.Data[k]; ok { + prev.Priority = max(prev.Priority, it.Priority) + prev.RetryAt = now + m.Data[k] = prev + continue + } + it.Attempts = 0 + it.RetryAt = now + it.EnqueuedAt = now + m.Data[k] = it + } + return nil +} + +func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + if m.Err != nil { + return nil, m.Err + } + var res []model.ArtworkQueueItem + now := time.Now() + for _, it := range m.Data { + if !it.RetryAt.After(now) { + res = append(res, it) + } + } + sort.Slice(res, func(i, j int) bool { + if res[i].Priority != res[j].Priority { + return res[i].Priority > res[j].Priority + } + return res[i].EnqueuedAt.Before(res[j].EnqueuedAt) + }) + if len(res) > n { + res = res[:n] + } + return res, nil +} + +func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + it, ok := m.Data[k] + if !ok { + return model.ErrNotFound + } + it.Attempts++ + it.RetryAt = retryAt + m.Data[k] = it + return nil +} + +func (m *MockArtworkQueueRepo) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + if it, ok := m.Data[k]; ok && it.RetryAt.Equal(seenRetryAt) { + it.Attempts++ + it.RetryAt = retryAt + m.Data[k] = it + } + return nil +} + +func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error { + if m.Err != nil { + return m.Err + } + delete(m.Data, iaKey(kind, id, imageType)) + return nil +} + +func (m *MockArtworkQueueRepo) DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + if it, ok := m.Data[k]; ok && it.RetryAt.Equal(retryAt) { + delete(m.Data, k) + } + return nil +} + +func (m *MockArtworkQueueRepo) PurgeDangling() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + var purged int64 + for k, it := range m.Data { + existing := m.ExistingIDs[it.ItemKind] + if existing == nil { + continue + } + if !existing[it.ItemID] { + delete(m.Data, k) + purged++ + } + } + return purged, nil +} + +func (m *MockArtworkQueueRepo) Count() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + return int64(len(m.Data)), nil +} + +func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + if m.Err != nil || m.ItemArtworkSource == nil { + return 0, m.Err + } + now := time.Now() + var inserted int64 + for _, ia := range m.ItemArtworkSource.ItemData { + if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) { + continue + } + k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType) + if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows + continue + } + m.Data[k] = model.ArtworkQueueItem{ + ItemKind: ia.ItemKind, + ItemID: ia.ItemID, + ImageType: ia.ImageType, + Priority: model.ArtworkPriorityRecheck, + RetryAt: now, + EnqueuedAt: now, + } + inserted++ + } + return inserted, nil +} diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go new file mode 100644 index 000000000..db73af84d --- /dev/null +++ b/tests/mock_artwork_repo.go @@ -0,0 +1,170 @@ +package tests + +import ( + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockArtworkRepo struct { + model.ArtworkRepository + Data map[string]model.Artwork + ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType) + OrphanHashes []string + Err error + // ExistingIDs, keyed by item_kind, backs PurgeDanglingItemArtwork; nil map keeps everything. + ExistingIDs map[string]map[string]bool +} + +func CreateMockArtworkRepo() *MockArtworkRepo { + return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}} +} + +func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType } + +func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) { + if m.Err != nil { + return nil, m.Err + } + if a, ok := m.Data[hash]; ok { + return &a, nil + } + return nil, model.ErrNotFound +} + +func (m *MockArtworkRepo) PutImage(a *model.Artwork) error { + if m.Err != nil { + return m.Err + } + // Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly. + a.CreatedAt = time.Now() + m.Data[a.Hash] = *a + return nil +} + +func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.Artwork{} + for _, h := range hashes { + if a, ok := m.Data[h]; ok { + res[h] = a + } + } + return res, nil +} + +func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) { + return m.OrphanHashes, m.Err +} + +func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) { + if m.Err != nil { + return nil, m.Err + } + mimes := make(map[string]string, len(m.Data)) + for h, a := range m.Data { + mimes[h] = a.Mime + } + return mimes, nil +} + +func (m *MockArtworkRepo) PurgeDanglingItemArtwork() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + var purged int64 + for k, ia := range m.ItemData { + // A nil per-kind map means that kind isn't tracked by the test, so keep it. + existing := m.ExistingIDs[ia.ItemKind] + if existing == nil { + continue + } + if !existing[ia.ItemID] { + delete(m.ItemData, k) + purged++ + } + } + return purged, nil +} + +func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error { + if m.Err != nil { + return m.Err + } + // Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted. + for _, h := range hashes { + if m.referenced(h) { + continue + } + if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) { + continue + } + delete(m.Data, h) + } + return nil +} + +func (m *MockArtworkRepo) referenced(hash string) bool { + for _, ia := range m.ItemData { + if ia.Hash == hash { + return true + } + } + return false +} + +func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { + if m.Err != nil { + return nil, m.Err + } + if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok { + return &ia, nil + } + return nil, model.ErrNotFound +} + +func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error { + if m.Err != nil { + return m.Err + } + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } + ia.UpdatedAt = time.Now() + if ia.AttemptedAt.IsZero() { + ia.AttemptedAt = ia.UpdatedAt + } + m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia + return nil +} + +func (m *MockArtworkRepo) DeleteForItem(kind, id string) error { + if m.Err != nil { + return m.Err + } + for k, ia := range m.ItemData { + if ia.ItemKind == kind && ia.ItemID == id { + delete(m.ItemData, k) + } + } + return nil +} + +func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.ItemArtworkInfo{} + for _, id := range ids { + if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok { + info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash} + if a, ok := m.Data[ia.Hash]; ok { + info.BlurHash = a.BlurHash + } + res[id] = info + } + } + return res, nil +} diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index e016a28de..d8585fabf 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -28,6 +28,8 @@ type MockDataStore struct { MockedScrobble model.ScrobbleRepository MockedRadio model.RadioRepository MockedPlugin model.PluginRepository + MockedArtwork model.ArtworkRepository + MockedArtworkQueue model.ArtworkQueueRepository scrobbleBufferMu sync.Mutex repoMu sync.Mutex @@ -247,6 +249,32 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository { return db.MockedPlugin } +func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository { + if db.MockedArtwork != nil { + return db.MockedArtwork + } + if db.RealDS != nil { + return db.RealDS.Artwork(ctx) + } + db.MockedArtwork = CreateMockArtworkRepo() + return db.MockedArtwork +} + +func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { + if db.MockedArtworkQueue != nil { + return db.MockedArtworkQueue + } + if db.RealDS != nil { + return db.RealDS.ArtworkQueue(ctx) + } + q := CreateMockArtworkQueueRepo() + if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok { + q.ItemArtworkSource = aw + } + db.MockedArtworkQueue = q + return db.MockedArtworkQueue +} + func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error { return block(db) } diff --git a/tests/mock_playlist_repo.go b/tests/mock_playlist_repo.go index 8f8842c8e..bb1fe3379 100644 --- a/tests/mock_playlist_repo.go +++ b/tests/mock_playlist_repo.go @@ -52,6 +52,18 @@ func (m *MockPlaylistRepo) GetAll(options ...model.QueryOptions) (model.Playlist return m.All, nil } +func (m *MockPlaylistRepo) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(options...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, p := range all { + ids[i] = p.ID + } + return ids, nil +} + func (m *MockPlaylistRepo) GetCursor(options ...model.QueryOptions) (model.PlaylistCursor, error) { res, err := m.GetAll(options...) if err != nil { diff --git a/tests/mock_playlist_track_repo.go b/tests/mock_playlist_track_repo.go index 2835baadd..83751bb28 100644 --- a/tests/mock_playlist_track_repo.go +++ b/tests/mock_playlist_track_repo.go @@ -14,6 +14,7 @@ type MockPlaylistTrackRepo struct { Reordered bool AddCount int Err error + AlbumIDs []string // stubbed result for GetAlbumIDs, ignoring options } func (m *MockPlaylistTrackRepo) SetData(tracks model.PlaylistTracks) { @@ -66,6 +67,13 @@ func (m *MockPlaylistTrackRepo) GetCursor(options ...model.QueryOptions) (model. }, nil } +func (m *MockPlaylistTrackRepo) GetAlbumIDs(...model.QueryOptions) ([]string, error) { + if m.Err != nil { + return nil, m.Err + } + return m.AlbumIDs, nil +} + func (m *MockPlaylistTrackRepo) GetMediaFileIDs(options ...model.QueryOptions) ([]string, error) { if m.Err != nil { return nil, m.Err diff --git a/tests/mock_radio_repository.go b/tests/mock_radio_repository.go index c50a529e5..6d1d45f6e 100644 --- a/tests/mock_radio_repository.go +++ b/tests/mock_radio_repository.go @@ -73,6 +73,18 @@ func (m *MockedRadioRepo) GetAll(qo ...model.QueryOptions) (model.Radios, error) return m.All, nil } +func (m *MockedRadioRepo) GetAllIDs(qo ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(qo...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, r := range all { + ids[i] = r.ID + } + return ids, nil +} + func (m *MockedRadioRepo) Put(radio *model.Radio, _ ...string) error { if m.Err { return errors.New("error")