From 7aacb01f4f5abd05a9b8ee9ef0b3c10161a2d951 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:31:03 -0400 Subject: [PATCH 01/51] feat(artwork): add artwork, item_artwork and artwork_queue tables --- .../20260722023032_add_artwork_tables.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 db/migrations/20260722023032_add_artwork_tables.sql diff --git a/db/migrations/20260722023032_add_artwork_tables.sql b/db/migrations/20260722023032_add_artwork_tables.sql new file mode 100644 index 000000000..94e0ebe5f --- /dev/null +++ b/db/migrations/20260722023032_add_artwork_tables.sql @@ -0,0 +1,41 @@ +-- +goose Up +CREATE TABLE artwork ( + hash TEXT PRIMARY KEY, + mime TEXT NOT NULL, + width INTEGER NOT NULL DEFAULT 0, + height INTEGER NOT NULL DEFAULT 0, + size_bytes INTEGER NOT NULL DEFAULT 0, + blur_hash TEXT NOT NULL DEFAULT '', + source_path TEXT NOT NULL DEFAULT '', + ref_mtime INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE item_artwork ( + item_kind TEXT NOT NULL, + item_id TEXT NOT NULL, + image_type TEXT NOT NULL DEFAULT 'primary', + hash TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + attempted_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY (item_kind, item_id, image_type) +) WITHOUT ROWID; +CREATE INDEX ix_item_artwork_hash ON item_artwork(hash); + +CREATE TABLE artwork_queue ( + item_kind TEXT NOT NULL, + item_id TEXT NOT NULL, + image_type TEXT NOT NULL DEFAULT 'primary', + priority INTEGER NOT NULL DEFAULT 0, + attempts INTEGER NOT NULL DEFAULT 0, + retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (item_kind, item_id, image_type) +) WITHOUT ROWID; +CREATE INDEX ix_artwork_queue_drain ON artwork_queue(retry_at, priority, enqueued_at); + +-- +goose Down +DROP TABLE artwork_queue; +DROP TABLE item_artwork; +DROP TABLE artwork; From 6cce65f7598f65e2437a3335f3021c813899ebda Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:35:34 -0400 Subject: [PATCH 02/51] feat(artwork): add artwork models, repository interfaces and mocks --- model/artwork.go | 85 +++++++++++++++++++++++ model/datastore.go | 3 + persistence/artwork_queue_repository.go | 43 ++++++++++++ persistence/artwork_repository.go | 43 ++++++++++++ persistence/item_artwork_repository.go | 43 ++++++++++++ persistence/persistence.go | 12 ++++ tests/mock_artwork_queue_repo.go | 91 +++++++++++++++++++++++++ tests/mock_artwork_repo.go | 65 ++++++++++++++++++ tests/mock_data_store.go | 36 ++++++++++ tests/mock_item_artwork_repo.go | 67 ++++++++++++++++++ 10 files changed, 488 insertions(+) create mode 100644 model/artwork.go create mode 100644 persistence/artwork_queue_repository.go create mode 100644 persistence/artwork_repository.go create mode 100644 persistence/item_artwork_repository.go create mode 100644 tests/mock_artwork_queue_repo.go create mode 100644 tests/mock_artwork_repo.go create mode 100644 tests/mock_item_artwork_repo.go diff --git a/model/artwork.go b/model/artwork.go new file mode 100644 index 000000000..dd6a3f436 --- /dev/null +++ b/model/artwork.go @@ -0,0 +1,85 @@ +package model + +import "time" + +// Artwork is one unique image, identified by the XXH3-64 hash of its bytes. +type Artwork struct { + Hash string `structs:"hash"` + Mime string `structs:"mime"` + Width int `structs:"width"` + Height int `structs:"height"` + SizeBytes int64 `structs:"size_bytes"` + BlurHash string `structs:"blur_hash"` + SourcePath string `structs:"source_path"` + RefMtime int64 `structs:"ref_mtime"` + CreatedAt time.Time `structs:"created_at"` +} + +const ImageTypePrimary = "primary" + +// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent. +type ItemArtwork struct { + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Hash string `structs:"hash"` + Source string `structs:"source"` + AttemptedAt time.Time `structs:"attempted_at"` + UpdatedAt time.Time `structs:"updated_at"` +} + +// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork). +type ItemArtworkInfo struct { + ItemID string + Hash string + BlurHash string + Absent bool +} + +type ArtworkQueueItem struct { + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Priority int `structs:"priority"` + Attempts int `structs:"attempts"` + RetryAt time.Time `structs:"retry_at"` + EnqueuedAt time.Time `structs:"enqueued_at"` +} + +// Queue priorities: higher drains first. +const ( + ArtworkPriorityRecheck = 0 + ArtworkPriorityBackfill = 10 + ArtworkPriorityScan = 50 + ArtworkPriorityBump = 100 +) + +type ArtworkRepository interface { + Get(hash string) (*Artwork, error) + Put(a *Artwork) error + GetBatch(hashes []string) (map[string]Artwork, error) + // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. + GetOrphanHashes(createdBefore time.Time) ([]string, error) + Delete(hashes ...string) error +} + +type ItemArtworkRepository interface { + Get(kind, id, imageType string) (*ItemArtwork, error) + Put(ia *ItemArtwork) error + DeleteForItem(kind, id string) error + // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. + GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) + // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. + EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) +} + +type ArtworkQueueRepository interface { + // Enqueue upserts; an existing row keeps the higher of the two priorities. + Enqueue(items ...ArtworkQueueItem) error + // DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc. + DequeueBatch(n int) ([]ArtworkQueueItem, error) + // MarkFailed increments attempts and pushes retry_at into the future. + MarkFailed(kind, id, imageType string, retryAt time.Time) error + Delete(kind, id, imageType string) error + Count() (int64, error) +} diff --git a/model/datastore.go b/model/datastore.go index 94c3c3622..c72eb9a96 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -40,6 +40,9 @@ type DataStore interface { ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository Scrobble(ctx context.Context) ScrobbleRepository Plugin(ctx context.Context) PluginRepository + Artwork(ctx context.Context) ArtworkRepository + ItemArtwork(ctx context.Context) ItemArtworkRepository + ArtworkQueue(ctx context.Context) ArtworkQueueRepository Resource(ctx context.Context, model any) ResourceRepository diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go new file mode 100644 index 000000000..4b166aa57 --- /dev/null +++ b/persistence/artwork_queue_repository.go @@ -0,0 +1,43 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +type artworkQueueRepository struct { + sqlRepository +} + +func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository { + r := &artworkQueueRepository{} + r.ctx = ctx + r.db = db + r.tableName = "artwork_queue" + return r +} + +func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error { + return nil +} + +func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + return nil, nil +} + +func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error { + return nil +} + +func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { + return nil +} + +func (r *artworkQueueRepository) Count() (int64, error) { + return 0, nil +} + +var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil) diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go new file mode 100644 index 000000000..4db47741b --- /dev/null +++ b/persistence/artwork_repository.go @@ -0,0 +1,43 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +type artworkRepository struct { + sqlRepository +} + +func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository { + r := &artworkRepository{} + r.ctx = ctx + r.db = db + r.tableName = "artwork" + return r +} + +func (r *artworkRepository) Get(hash string) (*model.Artwork, error) { + return nil, model.ErrNotFound +} + +func (r *artworkRepository) Put(a *model.Artwork) error { + return nil +} + +func (r *artworkRepository) GetBatch(hashes []string) (map[string]model.Artwork, error) { + return nil, nil +} + +func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) { + return nil, nil +} + +func (r *artworkRepository) Delete(hashes ...string) error { + return nil +} + +var _ model.ArtworkRepository = (*artworkRepository)(nil) diff --git a/persistence/item_artwork_repository.go b/persistence/item_artwork_repository.go new file mode 100644 index 000000000..8363264ca --- /dev/null +++ b/persistence/item_artwork_repository.go @@ -0,0 +1,43 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +type itemArtworkRepository struct { + sqlRepository +} + +func NewItemArtworkRepository(ctx context.Context, db dbx.Builder) model.ItemArtworkRepository { + r := &itemArtworkRepository{} + r.ctx = ctx + r.db = db + r.tableName = "item_artwork" + return r +} + +func (r *itemArtworkRepository) Get(kind, id, imageType string) (*model.ItemArtwork, error) { + return nil, model.ErrNotFound +} + +func (r *itemArtworkRepository) Put(ia *model.ItemArtwork) error { + return nil +} + +func (r *itemArtworkRepository) DeleteForItem(kind, id string) error { + return nil +} + +func (r *itemArtworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + return nil, nil +} + +func (r *itemArtworkRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + return 0, nil +} + +var _ model.ItemArtworkRepository = (*itemArtworkRepository)(nil) diff --git a/persistence/persistence.go b/persistence/persistence.go index 93f0e3e71..ed6d29865 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -97,6 +97,18 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository { return NewPluginRepository(ctx, s.getDBXBuilder()) } +func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository { + return NewArtworkRepository(ctx, s.getDBXBuilder()) +} + +func (s *SQLStore) ItemArtwork(ctx context.Context) model.ItemArtworkRepository { + return NewItemArtworkRepository(ctx, s.getDBXBuilder()) +} + +func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { + return NewArtworkQueueRepository(ctx, s.getDBXBuilder()) +} + func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository { switch m.(type) { case model.User: diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go new file mode 100644 index 000000000..b08a1297c --- /dev/null +++ b/tests/mock_artwork_queue_repo.go @@ -0,0 +1,91 @@ +package tests + +import ( + "sort" + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockArtworkQueueRepo struct { + model.ArtworkQueueRepository + Data map[string]model.ArtworkQueueItem // key: kind + "|" + id + "|" + imageType + Err error +} + +func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo { + return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}} +} + +func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { + if m.Err != nil { + return m.Err + } + for _, it := range items { + k := iaKey(it.ItemKind, it.ItemID, it.ImageType) + if prev, ok := m.Data[k]; ok && prev.Priority > it.Priority { + it.Priority = prev.Priority + } + if it.EnqueuedAt.IsZero() { + it.EnqueuedAt = time.Now() + } + if it.RetryAt.IsZero() { + it.RetryAt = time.Now() + } + m.Data[k] = it + } + return nil +} + +func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + if m.Err != nil { + return nil, m.Err + } + var res []model.ArtworkQueueItem + now := time.Now() + for _, it := range m.Data { + if !it.RetryAt.After(now) { + res = append(res, it) + } + } + sort.Slice(res, func(i, j int) bool { + if res[i].Priority != res[j].Priority { + return res[i].Priority > res[j].Priority + } + return res[i].EnqueuedAt.Before(res[j].EnqueuedAt) + }) + if len(res) > n { + res = res[:n] + } + return res, nil +} + +func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + it, ok := m.Data[k] + if !ok { + return model.ErrNotFound + } + it.Attempts++ + it.RetryAt = retryAt + m.Data[k] = it + return nil +} + +func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error { + if m.Err != nil { + return m.Err + } + delete(m.Data, iaKey(kind, id, imageType)) + return nil +} + +func (m *MockArtworkQueueRepo) Count() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + return int64(len(m.Data)), nil +} diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go new file mode 100644 index 000000000..c8b95a025 --- /dev/null +++ b/tests/mock_artwork_repo.go @@ -0,0 +1,65 @@ +package tests + +import ( + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockArtworkRepo struct { + model.ArtworkRepository + Data map[string]model.Artwork + Err error +} + +func CreateMockArtworkRepo() *MockArtworkRepo { + return &MockArtworkRepo{Data: map[string]model.Artwork{}} +} + +func (m *MockArtworkRepo) Get(hash string) (*model.Artwork, error) { + if m.Err != nil { + return nil, m.Err + } + if a, ok := m.Data[hash]; ok { + return &a, nil + } + return nil, model.ErrNotFound +} + +func (m *MockArtworkRepo) Put(a *model.Artwork) error { + if m.Err != nil { + return m.Err + } + if a.CreatedAt.IsZero() { + a.CreatedAt = time.Now() + } + m.Data[a.Hash] = *a + return nil +} + +func (m *MockArtworkRepo) GetBatch(hashes []string) (map[string]model.Artwork, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.Artwork{} + for _, h := range hashes { + if a, ok := m.Data[h]; ok { + res[h] = a + } + } + return res, nil +} + +func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) { + return nil, m.Err +} + +func (m *MockArtworkRepo) Delete(hashes ...string) error { + if m.Err != nil { + return m.Err + } + for _, h := range hashes { + delete(m.Data, h) + } + return nil +} diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index e016a28de..345a376e6 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -28,6 +28,9 @@ type MockDataStore struct { MockedScrobble model.ScrobbleRepository MockedRadio model.RadioRepository MockedPlugin model.PluginRepository + MockedArtwork model.ArtworkRepository + MockedItemArtwork model.ItemArtworkRepository + MockedArtworkQueue model.ArtworkQueueRepository scrobbleBufferMu sync.Mutex repoMu sync.Mutex @@ -247,6 +250,39 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository { return db.MockedPlugin } +func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository { + if db.MockedArtwork != nil { + return db.MockedArtwork + } + if db.RealDS != nil { + return db.RealDS.Artwork(ctx) + } + db.MockedArtwork = CreateMockArtworkRepo() + return db.MockedArtwork +} + +func (db *MockDataStore) ItemArtwork(ctx context.Context) model.ItemArtworkRepository { + if db.MockedItemArtwork != nil { + return db.MockedItemArtwork + } + if db.RealDS != nil { + return db.RealDS.ItemArtwork(ctx) + } + db.MockedItemArtwork = CreateMockItemArtworkRepo() + return db.MockedItemArtwork +} + +func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { + if db.MockedArtworkQueue != nil { + return db.MockedArtworkQueue + } + if db.RealDS != nil { + return db.RealDS.ArtworkQueue(ctx) + } + db.MockedArtworkQueue = CreateMockArtworkQueueRepo() + return db.MockedArtworkQueue +} + func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error { return block(db) } diff --git a/tests/mock_item_artwork_repo.go b/tests/mock_item_artwork_repo.go new file mode 100644 index 000000000..7d9cfbd9e --- /dev/null +++ b/tests/mock_item_artwork_repo.go @@ -0,0 +1,67 @@ +package tests + +import ( + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockItemArtworkRepo struct { + model.ItemArtworkRepository + Data map[string]model.ItemArtwork // key: kind + "|" + id + "|" + imageType + Err error +} + +func CreateMockItemArtworkRepo() *MockItemArtworkRepo { + return &MockItemArtworkRepo{Data: map[string]model.ItemArtwork{}} +} + +func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType } + +func (m *MockItemArtworkRepo) Get(kind, id, imageType string) (*model.ItemArtwork, error) { + if m.Err != nil { + return nil, m.Err + } + if ia, ok := m.Data[iaKey(kind, id, imageType)]; ok { + return &ia, nil + } + return nil, model.ErrNotFound +} + +func (m *MockItemArtworkRepo) Put(ia *model.ItemArtwork) error { + if m.Err != nil { + return m.Err + } + ia.UpdatedAt = time.Now() + m.Data[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia + return nil +} + +func (m *MockItemArtworkRepo) DeleteForItem(kind, id string) error { + if m.Err != nil { + return m.Err + } + for k, ia := range m.Data { + if ia.ItemKind == kind && ia.ItemID == id { + delete(m.Data, k) + } + } + return nil +} + +func (m *MockItemArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.ItemArtworkInfo{} + for _, id := range ids { + if ia, ok := m.Data[iaKey(kind, id, model.ImageTypePrimary)]; ok { + res[id] = model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash, Absent: ia.Hash == ""} + } + } + return res, nil +} + +func (m *MockItemArtworkRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + return 0, m.Err +} From f926539c044a87219e0c6be915ea4d2c2365f8b1 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:44:24 -0400 Subject: [PATCH 03/51] feat(artwork): implement artwork repository --- persistence/artwork_repository.go | 49 +++++++++++++-- persistence/artwork_repository_test.go | 85 ++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 persistence/artwork_repository_test.go diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 4db47741b..a49f30597 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -4,6 +4,7 @@ import ( "context" "time" + . "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/model" "github.com/pocketbase/dbx" ) @@ -21,23 +22,61 @@ func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepo } func (r *artworkRepository) Get(hash string) (*model.Artwork, error) { - return nil, model.ErrNotFound + sel := Select("*").From(r.tableName).Where(Eq{"hash": hash}) + var res model.Artwork + if err := r.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil } func (r *artworkRepository) Put(a *model.Artwork) error { - return nil + if a.CreatedAt.IsZero() { + a.CreatedAt = time.Now() + } + ins := Insert(r.tableName).SetMap(map[string]any{ + "hash": a.Hash, "mime": a.Mime, "width": a.Width, "height": a.Height, + "size_bytes": a.SizeBytes, "blur_hash": a.BlurHash, + "source_path": a.SourcePath, "ref_mtime": a.RefMtime, "created_at": a.CreatedAt, + }).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, + height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, + source_path=excluded.source_path, ref_mtime=excluded.ref_mtime`) + _, err := r.executeSQL(ins) + return err } func (r *artworkRepository) GetBatch(hashes []string) (map[string]model.Artwork, error) { - return nil, nil + res := map[string]model.Artwork{} + if len(hashes) == 0 { + return res, nil + } + sel := Select("*").From(r.tableName).Where(Eq{"hash": hashes}) + var all []model.Artwork + if err := r.queryAll(sel, &all); err != nil { + return nil, err + } + for _, a := range all { + res[a.Hash] = a + } + return res, nil } func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) { - return nil, nil + sel := Select("hash").From(r.tableName). + Where(And{ + Lt{"created_at": createdBefore}, + Expr("hash NOT IN (SELECT hash FROM item_artwork WHERE hash <> '')"), + }) + var hashes []string + err := r.queryAllSlice(sel, &hashes) + return hashes, err } func (r *artworkRepository) Delete(hashes ...string) error { - return nil + if len(hashes) == 0 { + return nil + } + return r.delete(Eq{"hash": hashes}) } var _ model.ArtworkRepository = (*artworkRepository)(nil) diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go new file mode 100644 index 000000000..e158754a0 --- /dev/null +++ b/persistence/artwork_repository_test.go @@ -0,0 +1,85 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state. +func clearArtworkTables() { + db := GetDBXBuilder() + for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} { + _, err := db.NewQuery("DELETE FROM " + t).Execute() + Expect(err).ToNot(HaveOccurred()) + } +} + +var _ = Describe("ArtworkRepository", func() { + var repo model.ArtworkRepository + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkRepository(context.Background(), GetDBXBuilder()) + }) + + It("stores and retrieves an artwork by hash", func() { + a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"} + Expect(repo.Put(a)).To(Succeed()) + + got, err := repo.Get("abc123") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Mime).To(Equal("image/jpeg")) + Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) + Expect(got.CreatedAt).ToNot(BeZero()) + }) + + It("is idempotent on Put (upsert by hash)", func() { + a := &model.Artwork{Hash: "dup1", Mime: "image/png"} + Expect(repo.Put(a)).To(Succeed()) + a.BlurHash = "XYZ" + Expect(repo.Put(a)).To(Succeed()) + got, _ := repo.Get("dup1") + Expect(got.BlurHash).To(Equal("XYZ")) + }) + + It("returns ErrNotFound for a missing hash", func() { + _, err := repo.Get("nope") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("fetches a batch", func() { + Expect(repo.Put(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.Put(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) + got, err := repo.GetBatch([]string{"b1", "b2", "missing"}) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got["b2"].Mime).To(Equal("image/png")) + }) + + It("finds orphans older than cutoff, honoring item_artwork references", func() { + Expect(repo.Put(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.Put(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) + iaRepo := NewItemArtworkRepository(context.Background(), GetDBXBuilder()) + Expect(iaRepo.Put(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) + + orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(ContainElement("orph1")) + Expect(orphans).ToNot(ContainElement("ref1")) + + orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(BeEmpty()) + }) + + It("deletes by hashes", func() { + Expect(repo.Put(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.Delete("d1")).To(Succeed()) + _, err := repo.Get("d1") + Expect(err).To(MatchError(model.ErrNotFound)) + }) +}) From 14dd57052ede5a55a493ff9b79d453c48b172218 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:45:03 -0400 Subject: [PATCH 04/51] feat(artwork): implement item_artwork repository with batched hydration --- persistence/item_artwork_repository.go | 60 +++++++++++++-- persistence/item_artwork_repository_test.go | 83 +++++++++++++++++++++ 2 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 persistence/item_artwork_repository_test.go diff --git a/persistence/item_artwork_repository.go b/persistence/item_artwork_repository.go index 8363264ca..08b4caf3c 100644 --- a/persistence/item_artwork_repository.go +++ b/persistence/item_artwork_repository.go @@ -4,6 +4,7 @@ import ( "context" "time" + . "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/model" "github.com/pocketbase/dbx" ) @@ -21,23 +22,72 @@ func NewItemArtworkRepository(ctx context.Context, db dbx.Builder) model.ItemArt } func (r *itemArtworkRepository) Get(kind, id, imageType string) (*model.ItemArtwork, error) { - return nil, model.ErrNotFound + sel := Select("*").From(r.tableName). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + var res model.ItemArtwork + if err := r.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil } func (r *itemArtworkRepository) Put(ia *model.ItemArtwork) error { - return nil + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } + ia.UpdatedAt = time.Now() + ins := Insert(r.tableName).SetMap(map[string]any{ + "item_kind": ia.ItemKind, "item_id": ia.ItemID, "image_type": ia.ImageType, + "hash": ia.Hash, "source": ia.Source, + "attempted_at": ia.AttemptedAt, "updated_at": ia.UpdatedAt, + }).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + hash=excluded.hash, source=excluded.source, + attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) + _, err := r.executeSQL(ins) + return err } func (r *itemArtworkRepository) DeleteForItem(kind, id string) error { - return nil + return r.delete(Eq{"item_kind": kind, "item_id": id}) } func (r *itemArtworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { - return nil, nil + res := map[string]model.ItemArtworkInfo{} + if len(ids) == 0 { + return res, nil + } + sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). + From(r.tableName + " ia"). + LeftJoin("artwork a ON a.hash = ia.hash"). + Where(And{ + Eq{"ia.item_kind": kind}, + Eq{"ia.image_type": model.ImageTypePrimary}, + Eq{"ia.item_id": ids}, + }) + var rows []struct { + ItemID string + Hash string + BlurHash string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + for _, row := range rows { + res[row.ItemID] = model.ItemArtworkInfo{ + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + } + } + return res, nil } func (r *itemArtworkRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { - return 0, nil + now := time.Now() + ins := Expr(`INSERT INTO artwork_queue (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) + SELECT item_kind, item_id, image_type, ?, 0, ?, ? + FROM item_artwork WHERE item_kind = ? AND hash = '' AND attempted_at < ? + ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, + model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) + return r.executeSQL(ins) } var _ model.ItemArtworkRepository = (*itemArtworkRepository)(nil) diff --git a/persistence/item_artwork_repository_test.go b/persistence/item_artwork_repository_test.go new file mode 100644 index 000000000..6aec55d89 --- /dev/null +++ b/persistence/item_artwork_repository_test.go @@ -0,0 +1,83 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ItemArtworkRepository", func() { + var repo model.ItemArtworkRepository + var awRepo model.ArtworkRepository + + BeforeEach(func() { + clearArtworkTables() + repo = NewItemArtworkRepository(context.Background(), GetDBXBuilder()) + awRepo = NewArtworkRepository(context.Background(), GetDBXBuilder()) + }) + + It("upserts and reads state", func() { + ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, + Hash: "h1", Source: "folder", AttemptedAt: time.Now()} + Expect(repo.Put(ia)).To(Succeed()) + ia.Source = "embedded" + Expect(repo.Put(ia)).To(Succeed()) + + got, err := repo.Get("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Source).To(Equal("embedded")) + Expect(got.UpdatedAt).ToNot(BeZero()) + }) + + It("represents known-absent as empty hash", func() { + Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", + ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + got, err := repo.Get("ar", "ar1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Hash).To(BeEmpty()) + }) + + It("hydrates a page in one batch, including blurhash and absence", func() { + Expect(awRepo.Put(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) + Expect(repo.Put(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) + Expect(repo.Put(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) + + info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(info).To(HaveLen(2)) + Expect(info["x1"].Hash).To(Equal("h9")) + Expect(info["x1"].BlurHash).To(Equal("BH9")) + Expect(info["x1"].Absent).To(BeFalse()) + Expect(info["x2"].Absent).To(BeTrue()) + _, unresolved := info["x3"] + Expect(unresolved).To(BeFalse()) + }) + + It("deletes all rows for an item", func() { + Expect(repo.Put(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) + Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) + _, err := repo.Get("pl", "p1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("enqueues stale absent states for recheck", func() { + old := time.Now().Add(-48 * time.Hour) + Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) + Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) + + n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + qRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + items, err := qRepo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(HaveLen(1)) + Expect(items[0].ItemID).To(Equal("stale1")) + Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) + }) +}) From fcff9c63e787012badd95232754c22ac55612cab Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:45:08 -0400 Subject: [PATCH 05/51] feat(artwork): implement artwork_queue repository --- persistence/artwork_queue_repository.go | 45 ++++++++++++-- persistence/artwork_queue_repository_test.go | 65 ++++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 persistence/artwork_queue_repository_test.go diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go index 4b166aa57..fdf88db24 100644 --- a/persistence/artwork_queue_repository.go +++ b/persistence/artwork_queue_repository.go @@ -4,6 +4,7 @@ import ( "context" "time" + . "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/model" "github.com/pocketbase/dbx" ) @@ -21,23 +22,59 @@ func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.Artwor } func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error { + now := time.Now() + for _, it := range items { + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary + } + ins := Insert(r.tableName).SetMap(map[string]any{ + "item_kind": it.ItemKind, "item_id": it.ItemID, "image_type": it.ImageType, + "priority": it.Priority, "attempts": 0, "retry_at": now, "enqueued_at": now, + }).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`) + if _, err := r.executeSQL(ins); err != nil { + return err + } + } return nil } func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { - return nil, nil + sel := Select("*").From(r.tableName). + Where(LtOrEq{"retry_at": time.Now()}). + OrderBy("priority DESC", "enqueued_at ASC"). + Limit(uint64(n)) + var res []model.ArtworkQueueItem + err := r.queryAll(sel, &res) + return res, err } func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error { - return nil + upd := Update(r.tableName). + Set("attempts", Expr("attempts + 1")). + Set("retry_at", retryAt). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + c, err := r.executeSQL(upd) + if err == nil && c == 0 { + return model.ErrNotFound + } + return err } func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { - return nil + return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) } func (r *artworkQueueRepository) Count() (int64, error) { - return 0, nil + sel := Select("count(*)").From(r.tableName) + var counts []int64 + if err := r.queryAllSlice(sel, &counts); err != nil { + return 0, err + } + if len(counts) == 0 { + return 0, nil + } + return counts[0], nil } var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil) diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go new file mode 100644 index 000000000..73665b07c --- /dev/null +++ b/persistence/artwork_queue_repository_test.go @@ -0,0 +1,65 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtworkQueueRepository", func() { + var repo model.ArtworkQueueRepository + + item := func(kind, id string, prio int) model.ArtworkQueueItem { + return model.ArtworkQueueItem{ItemKind: kind, ItemID: id, + ImageType: model.ImageTypePrimary, Priority: prio} + } + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + }) + + It("enqueues and dequeues by priority then FIFO", func() { + Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed()) + Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed()) + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got[0].ItemID).To(Equal("high")) + }) + + It("keeps the higher priority on duplicate enqueue", func() { + Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed()) + Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed()) + got, _ := repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump)) + }) + + It("hides failed items until retry_at", func() { + Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed()) + Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed()) + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeEmpty()) + + Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Attempts).To(Equal(2)) + }) + + It("deletes on completion and counts", func() { + Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed()) + n, _ := repo.Count() + Expect(n).To(Equal(int64(1))) + Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed()) + n, _ = repo.Count() + Expect(n).To(BeZero()) + }) +}) From b72597821a0bdbf2624985196bf16c516ed09a2b Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:53:00 -0400 Subject: [PATCH 06/51] feat(artwork): add content-addressed originals store --- core/artwork/originals/originals.go | 112 ++++++++++++++++++ .../artwork/originals/originals_suite_test.go | 17 +++ core/artwork/originals/originals_test.go | 80 +++++++++++++ go.mod | 2 +- 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 core/artwork/originals/originals.go create mode 100644 core/artwork/originals/originals_suite_test.go create mode 100644 core/artwork/originals/originals_test.go diff --git a/core/artwork/originals/originals.go b/core/artwork/originals/originals.go new file mode 100644 index 000000000..12cfe56cc --- /dev/null +++ b/core/artwork/originals/originals.go @@ -0,0 +1,112 @@ +// Package originals is the content-addressed store for artwork images that have +// no library file backing them (external downloads, embedded extractions, generated). +package originals + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/zeebo/xxh3" +) + +func Hash(r io.Reader) (string, error) { + d := xxh3.New() + if _, err := io.Copy(d, r); err != nil { + return "", err + } + return fmt.Sprintf("%016x", d.Sum64()), nil +} + +type Store struct { + root string +} + +func New(rootDir string) *Store { + return &Store{root: rootDir} +} + +// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into +// content-addressed paths and re-derived on Open, so they must be stable across OSes. +func extForMime(m string) string { + switch m { + case "image/jpeg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + } + return ".img" +} + +func (s *Store) path(hash, mimeType string) string { + return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType)) +} + +func (s *Store) Write(hash, mimeType string, r io.Reader) error { + dst := s.path(hash, mimeType) + if _, err := os.Stat(dst); err == nil { + return nil + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, r); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dst) +} + +func (s *Store) Open(hash, mimeType string) (io.ReadCloser, error) { + return os.Open(s.path(hash, mimeType)) +} + +func (s *Store) Remove(hash, mimeType string) error { + err := os.Remove(s.path(hash, mimeType)) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +func (s *Store) Sweep(keep func(hash string) bool) (int, error) { + removed := 0 + err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + name := d.Name() + if strings.HasPrefix(name, ".") { // in-flight temp files + return nil + } + hash := strings.TrimSuffix(name, filepath.Ext(name)) + if !keep(hash) { + // #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks + if err := os.Remove(path); err != nil { + return err + } + removed++ + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return removed, nil + } + return removed, err +} diff --git a/core/artwork/originals/originals_suite_test.go b/core/artwork/originals/originals_suite_test.go new file mode 100644 index 000000000..2686f9ec9 --- /dev/null +++ b/core/artwork/originals/originals_suite_test.go @@ -0,0 +1,17 @@ +package originals_test + +import ( + "testing" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOriginals(t *testing.T) { + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + RegisterFailHandler(Fail) + RunSpecs(t, "Originals Store Suite") +} diff --git a/core/artwork/originals/originals_test.go b/core/artwork/originals/originals_test.go new file mode 100644 index 000000000..caa861a6f --- /dev/null +++ b/core/artwork/originals/originals_test.go @@ -0,0 +1,80 @@ +package originals_test + +import ( + "bytes" + "io" + "os" + "path/filepath" + + "github.com/navidrome/navidrome/core/artwork/originals" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Originals Store", func() { + var store *originals.Store + var root string + + BeforeEach(func() { + root = GinkgoT().TempDir() + store = originals.New(root) + }) + + It("hashes deterministically", func() { + h1, err := originals.Hash(bytes.NewReader([]byte("some image bytes"))) + Expect(err).ToNot(HaveOccurred()) + h2, _ := originals.Hash(bytes.NewReader([]byte("some image bytes"))) + Expect(h1).To(Equal(h2)) + Expect(h1).To(HaveLen(16)) + h3, _ := originals.Hash(bytes.NewReader([]byte("other bytes"))) + Expect(h3).ToNot(Equal(h1)) + }) + + It("writes sharded and reads back", func() { + data := []byte("jpeg-bytes") + h, _ := originals.Hash(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + got, _ := io.ReadAll(rc) + Expect(got).To(Equal(data)) + }) + + It("is idempotent on duplicate writes", func() { + data := []byte("dup") + h, _ := originals.Hash(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + }) + + It("returns fs.ErrNotExist for missing images", func() { + _, err := store.Open("beefbeefbeefbeef", "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("removes without error when already gone", func() { + Expect(store.Remove("beefbeefbeefbeef", "image/jpeg")).To(Succeed()) + }) + + It("sweeps unknown files, keeps known ones", func() { + d1 := []byte("keep-me") + h1, _ := originals.Hash(bytes.NewReader(d1)) + Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed()) + d2 := []byte("orphan") + h2, _ := originals.Hash(bytes.NewReader(d2)) + Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed()) + + removed, err := store.Sweep(func(h string) bool { return h == h1 }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h2, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h1, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) +}) diff --git a/go.mod b/go.mod index 5488b41e4..b72f4035c 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/tetratelabs/wazero v1.12.0 github.com/unrolled/secure v1.17.0 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 + github.com/zeebo/xxh3 v1.1.0 go.senan.xyz/taglib v0.11.1 go.uber.org/goleak v1.3.0 golang.org/x/image v0.44.0 @@ -128,7 +129,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect github.com/valyala/fastjson v1.6.10 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect From db16b3de9a6f6eb5f0fe8087fc41e9a45e4329c4 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:58:20 -0400 Subject: [PATCH 07/51] feat(artwork): add artwork prune (orphan cleanup) --- core/artwork/prune.go | 48 ++++++++++++++++++++++++++++++ core/artwork/prune_test.go | 61 ++++++++++++++++++++++++++++++++++++++ tests/mock_artwork_repo.go | 7 +++-- 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 core/artwork/prune.go create mode 100644 core/artwork/prune_test.go diff --git a/core/artwork/prune.go b/core/artwork/prune.go new file mode 100644 index 000000000..7d66bd8fe --- /dev/null +++ b/core/artwork/prune.go @@ -0,0 +1,48 @@ +package artwork + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/core/artwork/originals" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// pruneMinAge guards the window between artwork insert and item_artwork upsert. +const pruneMinAge = time.Hour + +func Prune(ctx context.Context, ds model.DataStore, store *originals.Store) error { + repo := ds.Artwork(ctx) + orphans, err := repo.GetOrphanHashes(time.Now().Add(-pruneMinAge)) + if err != nil { + return err + } + if len(orphans) > 0 { + arts, err := repo.GetBatch(orphans) + if err != nil { + return err + } + if err := repo.Delete(orphans...); err != nil { + return err + } + for _, a := range arts { + if err := store.Remove(a.Hash, a.Mime); err != nil { + log.Warn(ctx, "Prune: could not remove artwork file", "hash", a.Hash, err) + } + } + log.Info(ctx, "Prune: removed orphan artwork", "count", len(orphans)) + } + + removed, err := store.Sweep(func(hash string) bool { + _, err := repo.Get(hash) + return err == nil + }) + if err != nil { + return err + } + if removed > 0 { + log.Info(ctx, "Prune: swept stray artwork files", "count", removed) + } + return nil +} diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go new file mode 100644 index 000000000..ae6901da4 --- /dev/null +++ b/core/artwork/prune_test.go @@ -0,0 +1,61 @@ +package artwork + +import ( + "bytes" + "context" + "os" + "time" + + "github.com/navidrome/navidrome/core/artwork/originals" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Prune", func() { + var ds *tests.MockDataStore + var store *originals.Store + var awRepo *tests.MockArtworkRepo + + BeforeEach(func() { + ds = &tests.MockDataStore{} + awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo) + store = originals.New(GinkgoT().TempDir()) + }) + + It("deletes orphan rows and their store files, keeps referenced ones", func() { + data := []byte("orphan-bytes") + h, _ := originals.Hash(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.Put(&model.Artwork{Hash: h, Mime: "image/jpeg", + CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) + awRepo.OrphanHashes = []string{h} + + kept := []byte("kept-bytes") + hk, _ := originals.Hash(bytes.NewReader(kept)) + Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed()) + Expect(awRepo.Put(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.Get(h) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = store.Open(h, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(hk, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("sweeps store files that have no artwork row", func() { + stray := []byte("no-row-bytes") + h, _ := originals.Hash(bytes.NewReader(stray)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := store.Open(h, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) +}) diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index c8b95a025..a30ffe97b 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -8,8 +8,9 @@ import ( type MockArtworkRepo struct { model.ArtworkRepository - Data map[string]model.Artwork - Err error + Data map[string]model.Artwork + OrphanHashes []string + Err error } func CreateMockArtworkRepo() *MockArtworkRepo { @@ -51,7 +52,7 @@ func (m *MockArtworkRepo) GetBatch(hashes []string) (map[string]model.Artwork, e } func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) { - return nil, m.Err + return m.OrphanHashes, m.Err } func (m *MockArtworkRepo) Delete(hashes ...string) error { From 3e7685adc23be7235d6d3c79ef2b48b683d74dfa Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 23:02:47 -0400 Subject: [PATCH 08/51] fix(artwork): never sweep files on transient DB errors during prune --- core/artwork/prune.go | 3 ++- core/artwork/prune_test.go | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index 7d66bd8fe..931990056 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -2,6 +2,7 @@ package artwork import ( "context" + "errors" "time" "github.com/navidrome/navidrome/core/artwork/originals" @@ -36,7 +37,7 @@ func Prune(ctx context.Context, ds model.DataStore, store *originals.Store) erro removed, err := store.Sweep(func(hash string) bool { _, err := repo.Get(hash) - return err == nil + return !errors.Is(err, model.ErrNotFound) }) if err != nil { return err diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index ae6901da4..54564a170 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -3,6 +3,7 @@ package artwork import ( "bytes" "context" + "errors" "os" "time" @@ -13,6 +14,14 @@ import ( . "github.com/onsi/gomega" ) +type flakyGetArtworkRepo struct { + *tests.MockArtworkRepo +} + +func (f *flakyGetArtworkRepo) Get(string) (*model.Artwork, error) { + return nil, errors.New("db locked") +} + var _ = Describe("Prune", func() { var ds *tests.MockDataStore var store *originals.Store @@ -58,4 +67,18 @@ var _ = Describe("Prune", func() { _, err := store.Open(h, "image/jpeg") Expect(os.IsNotExist(err)).To(BeTrue()) }) + + It("never sweeps files on a transient DB error", func() { + ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()} + + data := []byte("live-bytes") + h, _ := originals.Hash(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) }) From b16ef725c90bb40bdf69a8dc7229fc1c47314450 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 23:09:31 -0400 Subject: [PATCH 09/51] refactor(artwork): fold originals package into core/artwork as ImageStore --- .../originals.go => image_store.go} | 24 +++++++++---------- .../originals_test.go => image_store_test.go} | 23 +++++++++--------- .../artwork/originals/originals_suite_test.go | 17 ------------- core/artwork/prune.go | 3 +-- core/artwork/prune_test.go | 13 +++++----- 5 files changed, 30 insertions(+), 50 deletions(-) rename core/artwork/{originals/originals.go => image_store.go} (74%) rename core/artwork/{originals/originals_test.go => image_store_test.go} (76%) delete mode 100644 core/artwork/originals/originals_suite_test.go diff --git a/core/artwork/originals/originals.go b/core/artwork/image_store.go similarity index 74% rename from core/artwork/originals/originals.go rename to core/artwork/image_store.go index 12cfe56cc..c98064d13 100644 --- a/core/artwork/originals/originals.go +++ b/core/artwork/image_store.go @@ -1,6 +1,4 @@ -// Package originals is the content-addressed store for artwork images that have -// no library file backing them (external downloads, embedded extractions, generated). -package originals +package artwork import ( "errors" @@ -14,7 +12,7 @@ import ( "github.com/zeebo/xxh3" ) -func Hash(r io.Reader) (string, error) { +func HashImage(r io.Reader) (string, error) { d := xxh3.New() if _, err := io.Copy(d, r); err != nil { return "", err @@ -22,12 +20,14 @@ func Hash(r io.Reader) (string, error) { return fmt.Sprintf("%016x", d.Sum64()), nil } -type Store struct { +// ImageStore is the content-addressed store for artwork images that have no +// library file backing them (external downloads, embedded extractions, generated). +type ImageStore struct { root string } -func New(rootDir string) *Store { - return &Store{root: rootDir} +func NewImageStore(rootDir string) *ImageStore { + return &ImageStore{root: rootDir} } // extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into @@ -46,11 +46,11 @@ func extForMime(m string) string { return ".img" } -func (s *Store) path(hash, mimeType string) string { +func (s *ImageStore) path(hash, mimeType string) string { return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType)) } -func (s *Store) Write(hash, mimeType string, r io.Reader) error { +func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { dst := s.path(hash, mimeType) if _, err := os.Stat(dst); err == nil { return nil @@ -73,11 +73,11 @@ func (s *Store) Write(hash, mimeType string, r io.Reader) error { return os.Rename(tmp.Name(), dst) } -func (s *Store) Open(hash, mimeType string) (io.ReadCloser, error) { +func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) { return os.Open(s.path(hash, mimeType)) } -func (s *Store) Remove(hash, mimeType string) error { +func (s *ImageStore) Remove(hash, mimeType string) error { err := os.Remove(s.path(hash, mimeType)) if errors.Is(err, fs.ErrNotExist) { return nil @@ -85,7 +85,7 @@ func (s *Store) Remove(hash, mimeType string) error { return err } -func (s *Store) Sweep(keep func(hash string) bool) (int, error) { +func (s *ImageStore) Sweep(keep func(hash string) bool) (int, error) { removed := 0 err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { diff --git a/core/artwork/originals/originals_test.go b/core/artwork/image_store_test.go similarity index 76% rename from core/artwork/originals/originals_test.go rename to core/artwork/image_store_test.go index caa861a6f..c3ca69a58 100644 --- a/core/artwork/originals/originals_test.go +++ b/core/artwork/image_store_test.go @@ -1,4 +1,4 @@ -package originals_test +package artwork import ( "bytes" @@ -6,33 +6,32 @@ import ( "os" "path/filepath" - "github.com/navidrome/navidrome/core/artwork/originals" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -var _ = Describe("Originals Store", func() { - var store *originals.Store +var _ = Describe("ImageStore", func() { + var store *ImageStore var root string BeforeEach(func() { root = GinkgoT().TempDir() - store = originals.New(root) + store = NewImageStore(root) }) It("hashes deterministically", func() { - h1, err := originals.Hash(bytes.NewReader([]byte("some image bytes"))) + h1, err := HashImage(bytes.NewReader([]byte("some image bytes"))) Expect(err).ToNot(HaveOccurred()) - h2, _ := originals.Hash(bytes.NewReader([]byte("some image bytes"))) + h2, _ := HashImage(bytes.NewReader([]byte("some image bytes"))) Expect(h1).To(Equal(h2)) Expect(h1).To(HaveLen(16)) - h3, _ := originals.Hash(bytes.NewReader([]byte("other bytes"))) + h3, _ := HashImage(bytes.NewReader([]byte("other bytes"))) Expect(h3).ToNot(Equal(h1)) }) It("writes sharded and reads back", func() { data := []byte("jpeg-bytes") - h, _ := originals.Hash(bytes.NewReader(data)) + h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile()) @@ -46,7 +45,7 @@ var _ = Describe("Originals Store", func() { It("is idempotent on duplicate writes", func() { data := []byte("dup") - h, _ := originals.Hash(bytes.NewReader(data)) + h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) }) @@ -62,10 +61,10 @@ var _ = Describe("Originals Store", func() { It("sweeps unknown files, keeps known ones", func() { d1 := []byte("keep-me") - h1, _ := originals.Hash(bytes.NewReader(d1)) + h1, _ := HashImage(bytes.NewReader(d1)) Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed()) d2 := []byte("orphan") - h2, _ := originals.Hash(bytes.NewReader(d2)) + h2, _ := HashImage(bytes.NewReader(d2)) Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed()) removed, err := store.Sweep(func(h string) bool { return h == h1 }) diff --git a/core/artwork/originals/originals_suite_test.go b/core/artwork/originals/originals_suite_test.go deleted file mode 100644 index 2686f9ec9..000000000 --- a/core/artwork/originals/originals_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package originals_test - -import ( - "testing" - - "github.com/navidrome/navidrome/log" - "github.com/navidrome/navidrome/tests" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestOriginals(t *testing.T) { - tests.Init(t, false) - log.SetLevel(log.LevelFatal) - RegisterFailHandler(Fail) - RunSpecs(t, "Originals Store Suite") -} diff --git a/core/artwork/prune.go b/core/artwork/prune.go index 931990056..9c88611a3 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -5,7 +5,6 @@ import ( "errors" "time" - "github.com/navidrome/navidrome/core/artwork/originals" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" ) @@ -13,7 +12,7 @@ import ( // pruneMinAge guards the window between artwork insert and item_artwork upsert. const pruneMinAge = time.Hour -func Prune(ctx context.Context, ds model.DataStore, store *originals.Store) error { +func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { repo := ds.Artwork(ctx) orphans, err := repo.GetOrphanHashes(time.Now().Add(-pruneMinAge)) if err != nil { diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 54564a170..925e46fd1 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -7,7 +7,6 @@ import ( "os" "time" - "github.com/navidrome/navidrome/core/artwork/originals" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" @@ -24,25 +23,25 @@ func (f *flakyGetArtworkRepo) Get(string) (*model.Artwork, error) { var _ = Describe("Prune", func() { var ds *tests.MockDataStore - var store *originals.Store + var store *ImageStore var awRepo *tests.MockArtworkRepo BeforeEach(func() { ds = &tests.MockDataStore{} awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo) - store = originals.New(GinkgoT().TempDir()) + store = NewImageStore(GinkgoT().TempDir()) }) It("deletes orphan rows and their store files, keeps referenced ones", func() { data := []byte("orphan-bytes") - h, _ := originals.Hash(bytes.NewReader(data)) + h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) Expect(awRepo.Put(&model.Artwork{Hash: h, Mime: "image/jpeg", CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) awRepo.OrphanHashes = []string{h} kept := []byte("kept-bytes") - hk, _ := originals.Hash(bytes.NewReader(kept)) + hk, _ := HashImage(bytes.NewReader(kept)) Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed()) Expect(awRepo.Put(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed()) @@ -59,7 +58,7 @@ var _ = Describe("Prune", func() { It("sweeps store files that have no artwork row", func() { stray := []byte("no-row-bytes") - h, _ := originals.Hash(bytes.NewReader(stray)) + h, _ := HashImage(bytes.NewReader(stray)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed()) Expect(Prune(context.Background(), ds, store)).To(Succeed()) @@ -72,7 +71,7 @@ var _ = Describe("Prune", func() { ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()} data := []byte("live-bytes") - h, _ := originals.Hash(bytes.NewReader(data)) + h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) Expect(Prune(context.Background(), ds, store)).To(Succeed()) From 4f835437a9787f9900fc875b8d11056d49bb1c64 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 23:14:01 -0400 Subject: [PATCH 10/51] refactor(artwork): merge item artwork state into ArtworkRepository --- core/artwork/prune.go | 6 +- core/artwork/prune_test.go | 8 +- model/artwork.go | 20 ++-- model/datastore.go | 1 - persistence/artwork_repository.go | 79 ++++++++++++++- persistence/artwork_repository_test.go | 102 +++++++++++++++++--- persistence/item_artwork_repository.go | 93 ------------------ persistence/item_artwork_repository_test.go | 83 ---------------- persistence/persistence.go | 4 - tests/mock_artwork_repo.go | 61 +++++++++++- tests/mock_data_store.go | 12 --- tests/mock_item_artwork_repo.go | 67 ------------- 12 files changed, 232 insertions(+), 304 deletions(-) delete mode 100644 persistence/item_artwork_repository.go delete mode 100644 persistence/item_artwork_repository_test.go delete mode 100644 tests/mock_item_artwork_repo.go diff --git a/core/artwork/prune.go b/core/artwork/prune.go index 9c88611a3..11d9ce34f 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -19,11 +19,11 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { return err } if len(orphans) > 0 { - arts, err := repo.GetBatch(orphans) + arts, err := repo.GetImages(orphans) if err != nil { return err } - if err := repo.Delete(orphans...); err != nil { + if err := repo.DeleteImages(orphans...); err != nil { return err } for _, a := range arts { @@ -35,7 +35,7 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { } removed, err := store.Sweep(func(hash string) bool { - _, err := repo.Get(hash) + _, err := repo.GetImage(hash) return !errors.Is(err, model.ErrNotFound) }) if err != nil { diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 925e46fd1..68211879d 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -17,7 +17,7 @@ type flakyGetArtworkRepo struct { *tests.MockArtworkRepo } -func (f *flakyGetArtworkRepo) Get(string) (*model.Artwork, error) { +func (f *flakyGetArtworkRepo) GetImage(string) (*model.Artwork, error) { return nil, errors.New("db locked") } @@ -36,18 +36,18 @@ var _ = Describe("Prune", func() { data := []byte("orphan-bytes") h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) - Expect(awRepo.Put(&model.Artwork{Hash: h, Mime: "image/jpeg", + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) awRepo.OrphanHashes = []string{h} kept := []byte("kept-bytes") hk, _ := HashImage(bytes.NewReader(kept)) Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed()) - Expect(awRepo.Put(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed()) Expect(Prune(context.Background(), ds, store)).To(Succeed()) - _, err := awRepo.Get(h) + _, err := awRepo.GetImage(h) Expect(err).To(MatchError(model.ErrNotFound)) _, err = store.Open(h, "image/jpeg") Expect(os.IsNotExist(err)).To(BeTrue()) diff --git a/model/artwork.go b/model/artwork.go index dd6a3f436..8fe2bc7fe 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -55,21 +55,17 @@ const ( ) type ArtworkRepository interface { - Get(hash string) (*Artwork, error) - Put(a *Artwork) error - GetBatch(hashes []string) (map[string]Artwork, error) - // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. + // Image identity (artwork table) + GetImage(hash string) (*Artwork, error) + PutImage(a *Artwork) error + GetImages(hashes []string) (map[string]Artwork, error) GetOrphanHashes(createdBefore time.Time) ([]string, error) - Delete(hashes ...string) error -} - -type ItemArtworkRepository interface { - Get(kind, id, imageType string) (*ItemArtwork, error) - Put(ia *ItemArtwork) error + DeleteImages(hashes ...string) error + // Per-item state (item_artwork table) + GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error) + PutItemArtwork(ia *ItemArtwork) error DeleteForItem(kind, id string) error - // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) - // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) } diff --git a/model/datastore.go b/model/datastore.go index c72eb9a96..273ca714b 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -41,7 +41,6 @@ type DataStore interface { Scrobble(ctx context.Context) ScrobbleRepository Plugin(ctx context.Context) PluginRepository Artwork(ctx context.Context) ArtworkRepository - ItemArtwork(ctx context.Context) ItemArtworkRepository ArtworkQueue(ctx context.Context) ArtworkQueueRepository Resource(ctx context.Context, model any) ResourceRepository diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index a49f30597..839a390ed 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -21,7 +21,7 @@ func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepo return r } -func (r *artworkRepository) Get(hash string) (*model.Artwork, error) { +func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) { sel := Select("*").From(r.tableName).Where(Eq{"hash": hash}) var res model.Artwork if err := r.queryOne(sel, &res); err != nil { @@ -30,7 +30,7 @@ func (r *artworkRepository) Get(hash string) (*model.Artwork, error) { return &res, nil } -func (r *artworkRepository) Put(a *model.Artwork) error { +func (r *artworkRepository) PutImage(a *model.Artwork) error { if a.CreatedAt.IsZero() { a.CreatedAt = time.Now() } @@ -45,7 +45,7 @@ func (r *artworkRepository) Put(a *model.Artwork) error { return err } -func (r *artworkRepository) GetBatch(hashes []string) (map[string]model.Artwork, error) { +func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) { res := map[string]model.Artwork{} if len(hashes) == 0 { return res, nil @@ -72,11 +72,82 @@ func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, return hashes, err } -func (r *artworkRepository) Delete(hashes ...string) error { +func (r *artworkRepository) DeleteImages(hashes ...string) error { if len(hashes) == 0 { return nil } return r.delete(Eq{"hash": hashes}) } +func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { + sel := Select("*").From("item_artwork"). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + var res model.ItemArtwork + if err := r.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil +} + +func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error { + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } + ia.UpdatedAt = time.Now() + ins := Insert("item_artwork").SetMap(map[string]any{ + "item_kind": ia.ItemKind, "item_id": ia.ItemID, "image_type": ia.ImageType, + "hash": ia.Hash, "source": ia.Source, + "attempted_at": ia.AttemptedAt, "updated_at": ia.UpdatedAt, + }).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + hash=excluded.hash, source=excluded.source, + attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) + _, err := r.executeSQL(ins) + return err +} + +func (r *artworkRepository) DeleteForItem(kind, id string) error { + del := Delete("item_artwork").Where(Eq{"item_kind": kind, "item_id": id}) + _, err := r.executeSQL(del) + return err +} + +func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + res := map[string]model.ItemArtworkInfo{} + if len(ids) == 0 { + return res, nil + } + sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). + From("item_artwork ia"). + LeftJoin("artwork a ON a.hash = ia.hash"). + Where(And{ + Eq{"ia.item_kind": kind}, + Eq{"ia.image_type": model.ImageTypePrimary}, + Eq{"ia.item_id": ids}, + }) + var rows []struct { + ItemID string + Hash string + BlurHash string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + for _, row := range rows { + res[row.ItemID] = model.ItemArtworkInfo{ + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + } + } + return res, nil +} + +func (r *artworkRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + now := time.Now() + ins := Expr(`INSERT INTO artwork_queue (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) + SELECT item_kind, item_id, image_type, ?, 0, ?, ? + FROM item_artwork WHERE item_kind = ? AND hash = '' AND attempted_at < ? + ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, + model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) + return r.executeSQL(ins) +} + var _ model.ArtworkRepository = (*artworkRepository)(nil) diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index e158754a0..b4da4ebbe 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -28,9 +28,9 @@ var _ = Describe("ArtworkRepository", func() { It("stores and retrieves an artwork by hash", func() { a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"} - Expect(repo.Put(a)).To(Succeed()) + Expect(repo.PutImage(a)).To(Succeed()) - got, err := repo.Get("abc123") + got, err := repo.GetImage("abc123") Expect(err).ToNot(HaveOccurred()) Expect(got.Mime).To(Equal("image/jpeg")) Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) @@ -39,32 +39,31 @@ var _ = Describe("ArtworkRepository", func() { It("is idempotent on Put (upsert by hash)", func() { a := &model.Artwork{Hash: "dup1", Mime: "image/png"} - Expect(repo.Put(a)).To(Succeed()) + Expect(repo.PutImage(a)).To(Succeed()) a.BlurHash = "XYZ" - Expect(repo.Put(a)).To(Succeed()) - got, _ := repo.Get("dup1") + Expect(repo.PutImage(a)).To(Succeed()) + got, _ := repo.GetImage("dup1") Expect(got.BlurHash).To(Equal("XYZ")) }) It("returns ErrNotFound for a missing hash", func() { - _, err := repo.Get("nope") + _, err := repo.GetImage("nope") Expect(err).To(MatchError(model.ErrNotFound)) }) It("fetches a batch", func() { - Expect(repo.Put(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.Put(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) - got, err := repo.GetBatch([]string{"b1", "b2", "missing"}) + Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) + got, err := repo.GetImages([]string{"b1", "b2", "missing"}) Expect(err).ToNot(HaveOccurred()) Expect(got).To(HaveLen(2)) Expect(got["b2"].Mime).To(Equal("image/png")) }) It("finds orphans older than cutoff, honoring item_artwork references", func() { - Expect(repo.Put(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.Put(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) - iaRepo := NewItemArtworkRepository(context.Background(), GetDBXBuilder()) - Expect(iaRepo.Put(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute)) Expect(err).ToNot(HaveOccurred()) @@ -77,9 +76,80 @@ var _ = Describe("ArtworkRepository", func() { }) It("deletes by hashes", func() { - Expect(repo.Put(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.Delete("d1")).To(Succeed()) - _, err := repo.Get("d1") + Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.DeleteImages("d1")).To(Succeed()) + _, err := repo.GetImage("d1") Expect(err).To(MatchError(model.ErrNotFound)) }) }) + +var _ = Describe("ArtworkRepository item state", func() { + var repo model.ArtworkRepository + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkRepository(context.Background(), GetDBXBuilder()) + }) + + It("upserts and reads state", func() { + ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, + Hash: "h1", Source: "folder", AttemptedAt: time.Now()} + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + ia.Source = "embedded" + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + + got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Source).To(Equal("embedded")) + Expect(got.UpdatedAt).ToNot(BeZero()) + }) + + It("represents known-absent as empty hash", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", + ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Hash).To(BeEmpty()) + }) + + It("hydrates a page in one batch, including blurhash and absence", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) + + info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(info).To(HaveLen(2)) + Expect(info["x1"].Hash).To(Equal("h9")) + Expect(info["x1"].BlurHash).To(Equal("BH9")) + Expect(info["x1"].Absent).To(BeFalse()) + Expect(info["x2"].Absent).To(BeTrue()) + _, unresolved := info["x3"] + Expect(unresolved).To(BeFalse()) + }) + + It("deletes all rows for an item", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) + Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) + _, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("enqueues stale absent states for recheck", func() { + old := time.Now().Add(-48 * time.Hour) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) + + n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + qRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + items, err := qRepo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(HaveLen(1)) + Expect(items[0].ItemID).To(Equal("stale1")) + Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) + }) +}) diff --git a/persistence/item_artwork_repository.go b/persistence/item_artwork_repository.go deleted file mode 100644 index 08b4caf3c..000000000 --- a/persistence/item_artwork_repository.go +++ /dev/null @@ -1,93 +0,0 @@ -package persistence - -import ( - "context" - "time" - - . "github.com/Masterminds/squirrel" - "github.com/navidrome/navidrome/model" - "github.com/pocketbase/dbx" -) - -type itemArtworkRepository struct { - sqlRepository -} - -func NewItemArtworkRepository(ctx context.Context, db dbx.Builder) model.ItemArtworkRepository { - r := &itemArtworkRepository{} - r.ctx = ctx - r.db = db - r.tableName = "item_artwork" - return r -} - -func (r *itemArtworkRepository) Get(kind, id, imageType string) (*model.ItemArtwork, error) { - sel := Select("*").From(r.tableName). - Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) - var res model.ItemArtwork - if err := r.queryOne(sel, &res); err != nil { - return nil, err - } - return &res, nil -} - -func (r *itemArtworkRepository) Put(ia *model.ItemArtwork) error { - if ia.ImageType == "" { - ia.ImageType = model.ImageTypePrimary - } - ia.UpdatedAt = time.Now() - ins := Insert(r.tableName).SetMap(map[string]any{ - "item_kind": ia.ItemKind, "item_id": ia.ItemID, "image_type": ia.ImageType, - "hash": ia.Hash, "source": ia.Source, - "attempted_at": ia.AttemptedAt, "updated_at": ia.UpdatedAt, - }).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET - hash=excluded.hash, source=excluded.source, - attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) - _, err := r.executeSQL(ins) - return err -} - -func (r *itemArtworkRepository) DeleteForItem(kind, id string) error { - return r.delete(Eq{"item_kind": kind, "item_id": id}) -} - -func (r *itemArtworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { - res := map[string]model.ItemArtworkInfo{} - if len(ids) == 0 { - return res, nil - } - sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). - From(r.tableName + " ia"). - LeftJoin("artwork a ON a.hash = ia.hash"). - Where(And{ - Eq{"ia.item_kind": kind}, - Eq{"ia.image_type": model.ImageTypePrimary}, - Eq{"ia.item_id": ids}, - }) - var rows []struct { - ItemID string - Hash string - BlurHash string - } - if err := r.queryAll(sel, &rows); err != nil { - return nil, err - } - for _, row := range rows { - res[row.ItemID] = model.ItemArtworkInfo{ - ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", - } - } - return res, nil -} - -func (r *itemArtworkRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { - now := time.Now() - ins := Expr(`INSERT INTO artwork_queue (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) - SELECT item_kind, item_id, image_type, ?, 0, ?, ? - FROM item_artwork WHERE item_kind = ? AND hash = '' AND attempted_at < ? - ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, - model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) - return r.executeSQL(ins) -} - -var _ model.ItemArtworkRepository = (*itemArtworkRepository)(nil) diff --git a/persistence/item_artwork_repository_test.go b/persistence/item_artwork_repository_test.go deleted file mode 100644 index 6aec55d89..000000000 --- a/persistence/item_artwork_repository_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package persistence - -import ( - "context" - "time" - - "github.com/navidrome/navidrome/model" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("ItemArtworkRepository", func() { - var repo model.ItemArtworkRepository - var awRepo model.ArtworkRepository - - BeforeEach(func() { - clearArtworkTables() - repo = NewItemArtworkRepository(context.Background(), GetDBXBuilder()) - awRepo = NewArtworkRepository(context.Background(), GetDBXBuilder()) - }) - - It("upserts and reads state", func() { - ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, - Hash: "h1", Source: "folder", AttemptedAt: time.Now()} - Expect(repo.Put(ia)).To(Succeed()) - ia.Source = "embedded" - Expect(repo.Put(ia)).To(Succeed()) - - got, err := repo.Get("al", "al1", model.ImageTypePrimary) - Expect(err).ToNot(HaveOccurred()) - Expect(got.Source).To(Equal("embedded")) - Expect(got.UpdatedAt).ToNot(BeZero()) - }) - - It("represents known-absent as empty hash", func() { - Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", - ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) - got, err := repo.Get("ar", "ar1", model.ImageTypePrimary) - Expect(err).ToNot(HaveOccurred()) - Expect(got.Hash).To(BeEmpty()) - }) - - It("hydrates a page in one batch, including blurhash and absence", func() { - Expect(awRepo.Put(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) - Expect(repo.Put(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) - Expect(repo.Put(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) - - info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) - Expect(err).ToNot(HaveOccurred()) - Expect(info).To(HaveLen(2)) - Expect(info["x1"].Hash).To(Equal("h9")) - Expect(info["x1"].BlurHash).To(Equal("BH9")) - Expect(info["x1"].Absent).To(BeFalse()) - Expect(info["x2"].Absent).To(BeTrue()) - _, unresolved := info["x3"] - Expect(unresolved).To(BeFalse()) - }) - - It("deletes all rows for an item", func() { - Expect(repo.Put(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) - Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) - _, err := repo.Get("pl", "p1", model.ImageTypePrimary) - Expect(err).To(MatchError(model.ErrNotFound)) - }) - - It("enqueues stale absent states for recheck", func() { - old := time.Now().Add(-48 * time.Hour) - Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) - Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) - Expect(repo.Put(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) - - n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) - Expect(err).ToNot(HaveOccurred()) - Expect(n).To(Equal(int64(1))) - - qRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) - items, err := qRepo.DequeueBatch(10) - Expect(err).ToNot(HaveOccurred()) - Expect(items).To(HaveLen(1)) - Expect(items[0].ItemID).To(Equal("stale1")) - Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) - }) -}) diff --git a/persistence/persistence.go b/persistence/persistence.go index ed6d29865..9d3a33cfc 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -101,10 +101,6 @@ func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository { return NewArtworkRepository(ctx, s.getDBXBuilder()) } -func (s *SQLStore) ItemArtwork(ctx context.Context) model.ItemArtworkRepository { - return NewItemArtworkRepository(ctx, s.getDBXBuilder()) -} - func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { return NewArtworkQueueRepository(ctx, s.getDBXBuilder()) } diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index a30ffe97b..3989499ce 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -9,15 +9,18 @@ import ( type MockArtworkRepo struct { model.ArtworkRepository Data map[string]model.Artwork + ItemData map[string]model.ItemArtwork // key: kind + "|" + id + "|" + imageType OrphanHashes []string Err error } func CreateMockArtworkRepo() *MockArtworkRepo { - return &MockArtworkRepo{Data: map[string]model.Artwork{}} + return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}} } -func (m *MockArtworkRepo) Get(hash string) (*model.Artwork, error) { +func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType } + +func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) { if m.Err != nil { return nil, m.Err } @@ -27,7 +30,7 @@ func (m *MockArtworkRepo) Get(hash string) (*model.Artwork, error) { return nil, model.ErrNotFound } -func (m *MockArtworkRepo) Put(a *model.Artwork) error { +func (m *MockArtworkRepo) PutImage(a *model.Artwork) error { if m.Err != nil { return m.Err } @@ -38,7 +41,7 @@ func (m *MockArtworkRepo) Put(a *model.Artwork) error { return nil } -func (m *MockArtworkRepo) GetBatch(hashes []string) (map[string]model.Artwork, error) { +func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) { if m.Err != nil { return nil, m.Err } @@ -55,7 +58,7 @@ func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, er return m.OrphanHashes, m.Err } -func (m *MockArtworkRepo) Delete(hashes ...string) error { +func (m *MockArtworkRepo) DeleteImages(hashes ...string) error { if m.Err != nil { return m.Err } @@ -64,3 +67,51 @@ func (m *MockArtworkRepo) Delete(hashes ...string) error { } return nil } + +func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { + if m.Err != nil { + return nil, m.Err + } + if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok { + return &ia, nil + } + return nil, model.ErrNotFound +} + +func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error { + if m.Err != nil { + return m.Err + } + ia.UpdatedAt = time.Now() + m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia + return nil +} + +func (m *MockArtworkRepo) DeleteForItem(kind, id string) error { + if m.Err != nil { + return m.Err + } + for k, ia := range m.ItemData { + if ia.ItemKind == kind && ia.ItemID == id { + delete(m.ItemData, k) + } + } + return nil +} + +func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.ItemArtworkInfo{} + for _, id := range ids { + if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok { + res[id] = model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash, Absent: ia.Hash == ""} + } + } + return res, nil +} + +func (m *MockArtworkRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + return 0, m.Err +} diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index 345a376e6..a6ca29df5 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -29,7 +29,6 @@ type MockDataStore struct { MockedRadio model.RadioRepository MockedPlugin model.PluginRepository MockedArtwork model.ArtworkRepository - MockedItemArtwork model.ItemArtworkRepository MockedArtworkQueue model.ArtworkQueueRepository scrobbleBufferMu sync.Mutex repoMu sync.Mutex @@ -261,17 +260,6 @@ func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository { return db.MockedArtwork } -func (db *MockDataStore) ItemArtwork(ctx context.Context) model.ItemArtworkRepository { - if db.MockedItemArtwork != nil { - return db.MockedItemArtwork - } - if db.RealDS != nil { - return db.RealDS.ItemArtwork(ctx) - } - db.MockedItemArtwork = CreateMockItemArtworkRepo() - return db.MockedItemArtwork -} - func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { if db.MockedArtworkQueue != nil { return db.MockedArtworkQueue diff --git a/tests/mock_item_artwork_repo.go b/tests/mock_item_artwork_repo.go deleted file mode 100644 index 7d9cfbd9e..000000000 --- a/tests/mock_item_artwork_repo.go +++ /dev/null @@ -1,67 +0,0 @@ -package tests - -import ( - "time" - - "github.com/navidrome/navidrome/model" -) - -type MockItemArtworkRepo struct { - model.ItemArtworkRepository - Data map[string]model.ItemArtwork // key: kind + "|" + id + "|" + imageType - Err error -} - -func CreateMockItemArtworkRepo() *MockItemArtworkRepo { - return &MockItemArtworkRepo{Data: map[string]model.ItemArtwork{}} -} - -func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType } - -func (m *MockItemArtworkRepo) Get(kind, id, imageType string) (*model.ItemArtwork, error) { - if m.Err != nil { - return nil, m.Err - } - if ia, ok := m.Data[iaKey(kind, id, imageType)]; ok { - return &ia, nil - } - return nil, model.ErrNotFound -} - -func (m *MockItemArtworkRepo) Put(ia *model.ItemArtwork) error { - if m.Err != nil { - return m.Err - } - ia.UpdatedAt = time.Now() - m.Data[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia - return nil -} - -func (m *MockItemArtworkRepo) DeleteForItem(kind, id string) error { - if m.Err != nil { - return m.Err - } - for k, ia := range m.Data { - if ia.ItemKind == kind && ia.ItemID == id { - delete(m.Data, k) - } - } - return nil -} - -func (m *MockItemArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { - if m.Err != nil { - return nil, m.Err - } - res := map[string]model.ItemArtworkInfo{} - for _, id := range ids { - if ia, ok := m.Data[iaKey(kind, id, model.ImageTypePrimary)]; ok { - res[id] = model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash, Absent: ia.Hash == ""} - } - } - return res, nil -} - -func (m *MockItemArtworkRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { - return 0, m.Err -} From 1041e45ca7bfdbe04a8db61f87464ef9713792a6 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 23:24:42 -0400 Subject: [PATCH 11/51] fix(artwork): chunk unbounded IN clauses and restore interface docs --- model/artwork.go | 15 ++++-- persistence/artwork_repository.go | 71 +++++++++++++------------- persistence/artwork_repository_test.go | 15 ++++++ 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/model/artwork.go b/model/artwork.go index 8fe2bc7fe..6507c24d7 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -19,11 +19,13 @@ const ImageTypePrimary = "primary" // ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent. type ItemArtwork struct { - ItemKind string `structs:"item_kind"` - ItemID string `structs:"item_id"` - ImageType string `structs:"image_type"` - Hash string `structs:"hash"` - Source string `structs:"source"` + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Hash string `structs:"hash"` + Source string `structs:"source"` + // attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork; + // raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL. AttemptedAt time.Time `structs:"attempted_at"` UpdatedAt time.Time `structs:"updated_at"` } @@ -59,13 +61,16 @@ type ArtworkRepository interface { GetImage(hash string) (*Artwork, error) PutImage(a *Artwork) error GetImages(hashes []string) (map[string]Artwork, error) + // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. GetOrphanHashes(createdBefore time.Time) ([]string, error) DeleteImages(hashes ...string) error // Per-item state (item_artwork table) GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error) PutItemArtwork(ia *ItemArtwork) error DeleteForItem(kind, id string) error + // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) + // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) } diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 839a390ed..4556bc588 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "slices" "time" . "github.com/Masterminds/squirrel" @@ -47,16 +48,15 @@ func (r *artworkRepository) PutImage(a *model.Artwork) error { func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) { res := map[string]model.Artwork{} - if len(hashes) == 0 { - return res, nil - } - sel := Select("*").From(r.tableName).Where(Eq{"hash": hashes}) - var all []model.Artwork - if err := r.queryAll(sel, &all); err != nil { - return nil, err - } - for _, a := range all { - res[a.Hash] = a + for chunk := range slices.Chunk(hashes, 200) { + sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk}) + var all []model.Artwork + if err := r.queryAll(sel, &all); err != nil { + return nil, err + } + for _, a := range all { + res[a.Hash] = a + } } return res, nil } @@ -73,10 +73,12 @@ func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, } func (r *artworkRepository) DeleteImages(hashes ...string) error { - if len(hashes) == 0 { - return nil + for chunk := range slices.Chunk(hashes, 200) { + if err := r.delete(Eq{"hash": chunk}); err != nil { + return err + } } - return r.delete(Eq{"hash": hashes}) + return nil } func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { @@ -113,28 +115,27 @@ func (r *artworkRepository) DeleteForItem(kind, id string) error { func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { res := map[string]model.ItemArtworkInfo{} - if len(ids) == 0 { - return res, nil - } - sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). - From("item_artwork ia"). - LeftJoin("artwork a ON a.hash = ia.hash"). - Where(And{ - Eq{"ia.item_kind": kind}, - Eq{"ia.image_type": model.ImageTypePrimary}, - Eq{"ia.item_id": ids}, - }) - var rows []struct { - ItemID string - Hash string - BlurHash string - } - if err := r.queryAll(sel, &rows); err != nil { - return nil, err - } - for _, row := range rows { - res[row.ItemID] = model.ItemArtworkInfo{ - ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + for chunk := range slices.Chunk(ids, 200) { + sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). + From("item_artwork ia"). + LeftJoin("artwork a ON a.hash = ia.hash"). + Where(And{ + Eq{"ia.item_kind": kind}, + Eq{"ia.image_type": model.ImageTypePrimary}, + Eq{"ia.item_id": chunk}, + }) + var rows []struct { + ItemID string + Hash string + BlurHash string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + for _, row := range rows { + res[row.ItemID] = model.ItemArtworkInfo{ + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + } } } return res, nil diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index b4da4ebbe..a77fd7038 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "fmt" "time" "github.com/navidrome/navidrome/model" @@ -81,6 +82,20 @@ var _ = Describe("ArtworkRepository", func() { _, err := repo.GetImage("d1") Expect(err).To(MatchError(model.ErrNotFound)) }) + + It("fetches a batch larger than the SQL variable limit", func() { + hashes := make([]string, 0, 250) + for i := 0; i < 250; i++ { + h := fmt.Sprintf("big%03d", i) + Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + hashes = append(hashes, h) + } + hashes = append(hashes, "absent1", "absent2") + + got, err := repo.GetImages(hashes) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(250)) + }) }) var _ = Describe("ArtworkRepository item state", func() { From 8fd7ef19f385640206cf5b8c57c434fe9bffc080 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 23:37:05 -0400 Subject: [PATCH 12/51] refactor(artwork): apply simplify-pass cleanups Internal item_artwork sqlRepository helper, toSQLArgs upserts, batched queue enqueue, EnqueueStaleAbsent moved to queue repo, snapshot-based prune sweep, mock/real semantics aligned. --- core/artwork/prune.go | 13 +- core/artwork/prune_test.go | 4 +- model/artwork.go | 10 +- persistence/artwork_queue_repository.go | 42 ++-- persistence/artwork_queue_repository_test.go | 18 ++ persistence/artwork_repository.go | 76 +++--- persistence/artwork_repository_test.go | 247 +++++++++---------- tests/mock_artwork_queue_repo.go | 27 +- tests/mock_artwork_repo.go | 26 +- 9 files changed, 260 insertions(+), 203 deletions(-) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index 11d9ce34f..ebbb0bbc4 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -2,7 +2,6 @@ package artwork import ( "context" - "errors" "time" "github.com/navidrome/navidrome/log" @@ -34,9 +33,17 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { log.Info(ctx, "Prune: removed orphan artwork", "count", len(orphans)) } + hashes, err := repo.GetAllHashes() + if err != nil { + return err + } + known := make(map[string]struct{}, len(hashes)) + for _, h := range hashes { + known[h] = struct{}{} + } removed, err := store.Sweep(func(hash string) bool { - _, err := repo.GetImage(hash) - return !errors.Is(err, model.ErrNotFound) + _, ok := known[hash] + return ok }) if err != nil { return err diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 68211879d..2c8751674 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -17,7 +17,7 @@ type flakyGetArtworkRepo struct { *tests.MockArtworkRepo } -func (f *flakyGetArtworkRepo) GetImage(string) (*model.Artwork, error) { +func (f *flakyGetArtworkRepo) GetAllHashes() ([]string, error) { return nil, errors.New("db locked") } @@ -74,7 +74,7 @@ var _ = Describe("Prune", func() { h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) - Expect(Prune(context.Background(), ds, store)).To(Succeed()) + Expect(Prune(context.Background(), ds, store)).ToNot(Succeed()) rc, err := store.Open(h, "image/jpeg") Expect(err).ToNot(HaveOccurred()) diff --git a/model/artwork.go b/model/artwork.go index 6507c24d7..986a5c65d 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -35,9 +35,11 @@ type ItemArtworkInfo struct { ItemID string Hash string BlurHash string - Absent bool } +// Absent reports a known-absent artwork state (resolved, no image). +func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" } + type ArtworkQueueItem struct { ItemKind string `structs:"item_kind"` ItemID string `structs:"item_id"` @@ -70,8 +72,8 @@ type ArtworkRepository interface { DeleteForItem(kind, id string) error // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) - // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. - EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) + // GetAllHashes returns every stored artwork hash, for sweep membership checks. + GetAllHashes() ([]string, error) } type ArtworkQueueRepository interface { @@ -83,4 +85,6 @@ type ArtworkQueueRepository interface { MarkFailed(kind, id, imageType string, retryAt time.Time) error Delete(kind, id, imageType string) error Count() (int64, error) + // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. + EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) } diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go index fdf88db24..ced5e9913 100644 --- a/persistence/artwork_queue_repository.go +++ b/persistence/artwork_queue_repository.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "slices" "time" . "github.com/Masterminds/squirrel" @@ -9,6 +10,9 @@ import ( "github.com/pocketbase/dbx" ) +// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars). +const enqueueChunkSize = 100 + type artworkQueueRepository struct { sqlRepository } @@ -23,14 +27,15 @@ func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.Artwor func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error { now := time.Now() - for _, it := range items { - if it.ImageType == "" { - it.ImageType = model.ImageTypePrimary + for chunk := range slices.Chunk(items, enqueueChunkSize) { + ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at") + for _, it := range chunk { + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary + } + ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now) } - ins := Insert(r.tableName).SetMap(map[string]any{ - "item_kind": it.ItemKind, "item_id": it.ItemID, "image_type": it.ImageType, - "priority": it.Priority, "attempts": 0, "retry_at": now, "enqueued_at": now, - }).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`) if _, err := r.executeSQL(ins); err != nil { return err @@ -66,15 +71,20 @@ func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { } func (r *artworkQueueRepository) Count() (int64, error) { - sel := Select("count(*)").From(r.tableName) - var counts []int64 - if err := r.queryAllSlice(sel, &counts); err != nil { - return 0, err - } - if len(counts) == 0 { - return 0, nil - } - return counts[0], nil + var res struct{ Count int64 } + err := r.queryOne(Select("count(*) as count").From(r.tableName), &res) + return res.Count, err +} + +func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + now := time.Now() + // DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items. + ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) + SELECT item_kind, item_id, image_type, ?, 0, ?, ? + FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ? + ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, + model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) + return r.executeSQL(ins) } var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil) diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go index 73665b07c..2b6f0d8c5 100644 --- a/persistence/artwork_queue_repository_test.go +++ b/persistence/artwork_queue_repository_test.go @@ -62,4 +62,22 @@ var _ = Describe("ArtworkQueueRepository", func() { n, _ = repo.Count() Expect(n).To(BeZero()) }) + + It("enqueues stale absent states for recheck", func() { + awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) + old := time.Now().Add(-48 * time.Hour) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) + + n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + items, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(HaveLen(1)) + Expect(items[0].ItemID).To(Equal("stale1")) + Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) + }) }) diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 4556bc588..c87836ca9 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -10,8 +10,18 @@ import ( "github.com/pocketbase/dbx" ) +const ( + itemArtworkTable = "item_artwork" + artworkBatchSize = 200 +) + +type itemArtworkSQL struct { + sqlRepository +} + type artworkRepository struct { sqlRepository + items itemArtworkSQL } func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository { @@ -19,6 +29,9 @@ func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepo r.ctx = ctx r.db = db r.tableName = "artwork" + r.items.ctx = ctx + r.items.db = db + r.items.tableName = itemArtworkTable return r } @@ -35,20 +48,20 @@ func (r *artworkRepository) PutImage(a *model.Artwork) error { if a.CreatedAt.IsZero() { a.CreatedAt = time.Now() } - ins := Insert(r.tableName).SetMap(map[string]any{ - "hash": a.Hash, "mime": a.Mime, "width": a.Width, "height": a.Height, - "size_bytes": a.SizeBytes, "blur_hash": a.BlurHash, - "source_path": a.SourcePath, "ref_mtime": a.RefMtime, "created_at": a.CreatedAt, - }).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, + values, err := toSQLArgs(*a) + if err != nil { + return err + } + ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime`) - _, err := r.executeSQL(ins) + _, err = r.executeSQL(ins) return err } func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) { res := map[string]model.Artwork{} - for chunk := range slices.Chunk(hashes, 200) { + for chunk := range slices.Chunk(hashes, artworkBatchSize) { sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk}) var all []model.Artwork if err := r.queryAll(sel, &all); err != nil { @@ -61,11 +74,18 @@ func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork return res, nil } +func (r *artworkRepository) GetAllHashes() ([]string, error) { + sel := Select("hash").From(r.tableName) + var hashes []string + err := r.queryAllSlice(sel, &hashes) + return hashes, err +} + func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) { sel := Select("hash").From(r.tableName). Where(And{ Lt{"created_at": createdBefore}, - Expr("hash NOT IN (SELECT hash FROM item_artwork WHERE hash <> '')"), + Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"), }) var hashes []string err := r.queryAllSlice(sel, &hashes) @@ -73,7 +93,7 @@ func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, } func (r *artworkRepository) DeleteImages(hashes ...string) error { - for chunk := range slices.Chunk(hashes, 200) { + for chunk := range slices.Chunk(hashes, artworkBatchSize) { if err := r.delete(Eq{"hash": chunk}); err != nil { return err } @@ -82,10 +102,10 @@ func (r *artworkRepository) DeleteImages(hashes ...string) error { } func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { - sel := Select("*").From("item_artwork"). + sel := Select("*").From(itemArtworkTable). Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) var res model.ItemArtwork - if err := r.queryOne(sel, &res); err != nil { + if err := r.items.queryOne(sel, &res); err != nil { return nil, err } return &res, nil @@ -96,28 +116,26 @@ func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error { ia.ImageType = model.ImageTypePrimary } ia.UpdatedAt = time.Now() - ins := Insert("item_artwork").SetMap(map[string]any{ - "item_kind": ia.ItemKind, "item_id": ia.ItemID, "image_type": ia.ImageType, - "hash": ia.Hash, "source": ia.Source, - "attempted_at": ia.AttemptedAt, "updated_at": ia.UpdatedAt, - }).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + values, err := toSQLArgs(*ia) + if err != nil { + return err + } + ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET hash=excluded.hash, source=excluded.source, attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) - _, err := r.executeSQL(ins) + _, err = r.items.executeSQL(ins) return err } func (r *artworkRepository) DeleteForItem(kind, id string) error { - del := Delete("item_artwork").Where(Eq{"item_kind": kind, "item_id": id}) - _, err := r.executeSQL(del) - return err + return r.items.delete(Eq{"item_kind": kind, "item_id": id}) } func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { res := map[string]model.ItemArtworkInfo{} - for chunk := range slices.Chunk(ids, 200) { + for chunk := range slices.Chunk(ids, artworkBatchSize) { sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). - From("item_artwork ia"). + From(itemArtworkTable + " ia"). LeftJoin("artwork a ON a.hash = ia.hash"). Where(And{ Eq{"ia.item_kind": kind}, @@ -129,26 +147,16 @@ func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[stri Hash string BlurHash string } - if err := r.queryAll(sel, &rows); err != nil { + if err := r.items.queryAll(sel, &rows); err != nil { return nil, err } for _, row := range rows { res[row.ItemID] = model.ItemArtworkInfo{ - ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, } } } return res, nil } -func (r *artworkRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { - now := time.Now() - ins := Expr(`INSERT INTO artwork_queue (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) - SELECT item_kind, item_id, image_type, ?, 0, ?, ? - FROM item_artwork WHERE item_kind = ? AND hash = '' AND attempted_at < ? - ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, - model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) - return r.executeSQL(ins) -} - var _ model.ArtworkRepository = (*artworkRepository)(nil) diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index a77fd7038..0fdb122b6 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -27,144 +27,129 @@ var _ = Describe("ArtworkRepository", func() { repo = NewArtworkRepository(context.Background(), GetDBXBuilder()) }) - It("stores and retrieves an artwork by hash", func() { - a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"} - Expect(repo.PutImage(a)).To(Succeed()) + Context("image identity", func() { + It("stores and retrieves an artwork by hash", func() { + a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"} + Expect(repo.PutImage(a)).To(Succeed()) - got, err := repo.GetImage("abc123") - Expect(err).ToNot(HaveOccurred()) - Expect(got.Mime).To(Equal("image/jpeg")) - Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) - Expect(got.CreatedAt).ToNot(BeZero()) + got, err := repo.GetImage("abc123") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Mime).To(Equal("image/jpeg")) + Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) + Expect(got.CreatedAt).ToNot(BeZero()) + }) + + It("is idempotent on Put (upsert by hash)", func() { + a := &model.Artwork{Hash: "dup1", Mime: "image/png"} + Expect(repo.PutImage(a)).To(Succeed()) + a.BlurHash = "XYZ" + Expect(repo.PutImage(a)).To(Succeed()) + got, _ := repo.GetImage("dup1") + Expect(got.BlurHash).To(Equal("XYZ")) + }) + + It("returns ErrNotFound for a missing hash", func() { + _, err := repo.GetImage("nope") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("fetches a batch", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) + got, err := repo.GetImages([]string{"b1", "b2", "missing"}) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got["b2"].Mime).To(Equal("image/png")) + }) + + It("returns every stored hash", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed()) + hashes, err := repo.GetAllHashes() + Expect(err).ToNot(HaveOccurred()) + Expect(hashes).To(ConsistOf("all1", "all2")) + }) + + It("finds orphans older than cutoff, honoring item_artwork references", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) + + orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(ContainElement("orph1")) + Expect(orphans).ToNot(ContainElement("ref1")) + + orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(BeEmpty()) + }) + + It("deletes by hashes", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.DeleteImages("d1")).To(Succeed()) + _, err := repo.GetImage("d1") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("fetches a batch larger than the SQL variable limit", func() { + hashes := make([]string, 0, 250) + for i := range 250 { + h := fmt.Sprintf("big%03d", i) + Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + hashes = append(hashes, h) + } + hashes = append(hashes, "absent1", "absent2") + + got, err := repo.GetImages(hashes) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(250)) + }) }) - It("is idempotent on Put (upsert by hash)", func() { - a := &model.Artwork{Hash: "dup1", Mime: "image/png"} - Expect(repo.PutImage(a)).To(Succeed()) - a.BlurHash = "XYZ" - Expect(repo.PutImage(a)).To(Succeed()) - got, _ := repo.GetImage("dup1") - Expect(got.BlurHash).To(Equal("XYZ")) - }) + Context("item state", func() { + It("upserts and reads state", func() { + ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, + Hash: "h1", Source: "folder", AttemptedAt: time.Now()} + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + ia.Source = "embedded" + Expect(repo.PutItemArtwork(ia)).To(Succeed()) - It("returns ErrNotFound for a missing hash", func() { - _, err := repo.GetImage("nope") - Expect(err).To(MatchError(model.ErrNotFound)) - }) + got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Source).To(Equal("embedded")) + Expect(got.UpdatedAt).ToNot(BeZero()) + }) - It("fetches a batch", func() { - Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) - got, err := repo.GetImages([]string{"b1", "b2", "missing"}) - Expect(err).ToNot(HaveOccurred()) - Expect(got).To(HaveLen(2)) - Expect(got["b2"].Mime).To(Equal("image/png")) - }) + It("represents known-absent as empty hash", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", + ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Hash).To(BeEmpty()) + }) - It("finds orphans older than cutoff, honoring item_artwork references", func() { - Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) + It("hydrates a page in one batch, including blurhash and absence", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) - orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute)) - Expect(err).ToNot(HaveOccurred()) - Expect(orphans).To(ContainElement("orph1")) - Expect(orphans).ToNot(ContainElement("ref1")) + info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(info).To(HaveLen(2)) + Expect(info["x1"].Hash).To(Equal("h9")) + Expect(info["x1"].BlurHash).To(Equal("BH9")) + Expect(info["x1"].Absent()).To(BeFalse()) + Expect(info["x2"].Absent()).To(BeTrue()) + _, unresolved := info["x3"] + Expect(unresolved).To(BeFalse()) + }) - orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour)) - Expect(err).ToNot(HaveOccurred()) - Expect(orphans).To(BeEmpty()) - }) - - It("deletes by hashes", func() { - Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.DeleteImages("d1")).To(Succeed()) - _, err := repo.GetImage("d1") - Expect(err).To(MatchError(model.ErrNotFound)) - }) - - It("fetches a batch larger than the SQL variable limit", func() { - hashes := make([]string, 0, 250) - for i := 0; i < 250; i++ { - h := fmt.Sprintf("big%03d", i) - Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) - hashes = append(hashes, h) - } - hashes = append(hashes, "absent1", "absent2") - - got, err := repo.GetImages(hashes) - Expect(err).ToNot(HaveOccurred()) - Expect(got).To(HaveLen(250)) - }) -}) - -var _ = Describe("ArtworkRepository item state", func() { - var repo model.ArtworkRepository - - BeforeEach(func() { - clearArtworkTables() - repo = NewArtworkRepository(context.Background(), GetDBXBuilder()) - }) - - It("upserts and reads state", func() { - ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, - Hash: "h1", Source: "folder", AttemptedAt: time.Now()} - Expect(repo.PutItemArtwork(ia)).To(Succeed()) - ia.Source = "embedded" - Expect(repo.PutItemArtwork(ia)).To(Succeed()) - - got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary) - Expect(err).ToNot(HaveOccurred()) - Expect(got.Source).To(Equal("embedded")) - Expect(got.UpdatedAt).ToNot(BeZero()) - }) - - It("represents known-absent as empty hash", func() { - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", - ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) - got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary) - Expect(err).ToNot(HaveOccurred()) - Expect(got.Hash).To(BeEmpty()) - }) - - It("hydrates a page in one batch, including blurhash and absence", func() { - Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) - - info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) - Expect(err).ToNot(HaveOccurred()) - Expect(info).To(HaveLen(2)) - Expect(info["x1"].Hash).To(Equal("h9")) - Expect(info["x1"].BlurHash).To(Equal("BH9")) - Expect(info["x1"].Absent).To(BeFalse()) - Expect(info["x2"].Absent).To(BeTrue()) - _, unresolved := info["x3"] - Expect(unresolved).To(BeFalse()) - }) - - It("deletes all rows for an item", func() { - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) - Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) - _, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary) - Expect(err).To(MatchError(model.ErrNotFound)) - }) - - It("enqueues stale absent states for recheck", func() { - old := time.Now().Add(-48 * time.Hour) - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) - Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) - - n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) - Expect(err).ToNot(HaveOccurred()) - Expect(n).To(Equal(int64(1))) - - qRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) - items, err := qRepo.DequeueBatch(10) - Expect(err).ToNot(HaveOccurred()) - Expect(items).To(HaveLen(1)) - Expect(items[0].ItemID).To(Equal("stale1")) - Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) + It("deletes all rows for an item", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) + Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) + _, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) }) }) diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go index b08a1297c..dcec7486d 100644 --- a/tests/mock_artwork_queue_repo.go +++ b/tests/mock_artwork_queue_repo.go @@ -9,7 +9,7 @@ import ( type MockArtworkQueueRepo struct { model.ArtworkQueueRepository - Data map[string]model.ArtworkQueueItem // key: kind + "|" + id + "|" + imageType + Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType) Err error } @@ -21,16 +21,23 @@ func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { if m.Err != nil { return m.Err } + now := time.Now() for _, it := range items { - k := iaKey(it.ItemKind, it.ItemID, it.ImageType) - if prev, ok := m.Data[k]; ok && prev.Priority > it.Priority { - it.Priority = prev.Priority - } - if it.EnqueuedAt.IsZero() { - it.EnqueuedAt = time.Now() + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary } if it.RetryAt.IsZero() { - it.RetryAt = time.Now() + it.RetryAt = now + } + k := iaKey(it.ItemKind, it.ItemID, it.ImageType) + if prev, ok := m.Data[k]; ok { + prev.Priority = max(prev.Priority, it.Priority) + prev.RetryAt = it.RetryAt + m.Data[k] = prev + continue + } + if it.EnqueuedAt.IsZero() { + it.EnqueuedAt = now } m.Data[k] = it } @@ -89,3 +96,7 @@ func (m *MockArtworkQueueRepo) Count() (int64, error) { } return int64(len(m.Data)), nil } + +func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + return 0, m.Err +} diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index 3989499ce..8ac4820e6 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -9,7 +9,7 @@ import ( type MockArtworkRepo struct { model.ArtworkRepository Data map[string]model.Artwork - ItemData map[string]model.ItemArtwork // key: kind + "|" + id + "|" + imageType + ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType) OrphanHashes []string Err error } @@ -58,6 +58,17 @@ func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, er return m.OrphanHashes, m.Err } +func (m *MockArtworkRepo) GetAllHashes() ([]string, error) { + if m.Err != nil { + return nil, m.Err + } + hashes := make([]string, 0, len(m.Data)) + for h := range m.Data { + hashes = append(hashes, h) + } + return hashes, nil +} + func (m *MockArtworkRepo) DeleteImages(hashes ...string) error { if m.Err != nil { return m.Err @@ -82,6 +93,9 @@ func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error { if m.Err != nil { return m.Err } + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } ia.UpdatedAt = time.Now() m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia return nil @@ -106,12 +120,12 @@ func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string res := map[string]model.ItemArtworkInfo{} for _, id := range ids { if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok { - res[id] = model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash, Absent: ia.Hash == ""} + info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash} + if a, ok := m.Data[ia.Hash]; ok { + info.BlurHash = a.BlurHash + } + res[id] = info } } return res, nil } - -func (m *MockArtworkRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { - return 0, m.Err -} From 6f7f9c64633e362e5f175ec475d76eb879752449 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 23:57:58 -0400 Subject: [PATCH 13/51] fix(artwork): address review findings on prune/sweep races and mock fidelity Sweep now honors an mtime grace window (in-flight acquisitions and temp files), reacquired orphans reset the prune grace window, and the queue mock implements real stale-absent semantics. --- core/artwork/image_store.go | 20 +++++++++++---- core/artwork/image_store_test.go | 35 +++++++++++++++++++++++++- core/artwork/prune.go | 7 ++++-- core/artwork/prune_test.go | 2 ++ persistence/artwork_repository.go | 8 +++--- persistence/artwork_repository_test.go | 14 +++++++++++ tests/mock_artwork_queue_repo.go | 27 +++++++++++++++++++- tests/mock_data_store.go | 6 ++++- 8 files changed, 105 insertions(+), 14 deletions(-) diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go index c98064d13..517dfc879 100644 --- a/core/artwork/image_store.go +++ b/core/artwork/image_store.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/zeebo/xxh3" ) @@ -85,18 +86,27 @@ func (s *ImageStore) Remove(hash, mimeType string) error { return err } -func (s *ImageStore) Sweep(keep func(hash string) bool) (int, error) { +// Sweep removes store files not accepted by keep. Files modified after cutoff +// (including temp files) are always kept: their acquisition row may not be committed yet. +func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash string) bool) (int, error) { removed := 0 err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { return err } - name := d.Name() - if strings.HasPrefix(name, ".") { // in-flight temp files + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().After(cutoff) { return nil } - hash := strings.TrimSuffix(name, filepath.Ext(name)) - if !keep(hash) { + name := d.Name() + remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window + if !remove { + remove = !keep(strings.TrimSuffix(name, filepath.Ext(name))) + } + if remove { // #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks if err := os.Remove(path); err != nil { return err diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go index c3ca69a58..2647f3b7f 100644 --- a/core/artwork/image_store_test.go +++ b/core/artwork/image_store_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -67,7 +68,10 @@ var _ = Describe("ImageStore", func() { h2, _ := HashImage(bytes.NewReader(d2)) Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed()) - removed, err := store.Sweep(func(h string) bool { return h == h1 }) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h string) bool { return h == h1 }) Expect(err).ToNot(HaveOccurred()) Expect(removed).To(Equal(1)) _, err = store.Open(h2, "image/jpeg") @@ -76,4 +80,33 @@ var _ = Describe("ImageStore", func() { Expect(err).ToNot(HaveOccurred()) rc.Close() }) + + It("keeps young unknown files inside the grace window", func() { + d := []byte("fresh-orphan") + h, _ := HashImage(bytes.NewReader(d)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string) bool { return false }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(0)) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("removes abandoned temp files past the grace window, keeps fresh ones", func() { + oldTmp := filepath.Join(root, ".old.tmp") + Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(oldTmp, old, old)).To(Succeed()) + + freshTmp := filepath.Join(root, ".fresh.tmp") + Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string) bool { return true }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + Expect(oldTmp).ToNot(BeAnExistingFile()) + Expect(freshTmp).To(BeAnExistingFile()) + }) }) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index ebbb0bbc4..b787dd1c9 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -13,7 +13,10 @@ const pruneMinAge = time.Hour func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { repo := ds.Artwork(ctx) - orphans, err := repo.GetOrphanHashes(time.Now().Add(-pruneMinAge)) + // One grace cutoff for both the DB orphan check and the file sweep: files younger + // than the window may belong to acquisitions whose rows aren't committed yet. + cutoff := time.Now().Add(-pruneMinAge) + orphans, err := repo.GetOrphanHashes(cutoff) if err != nil { return err } @@ -41,7 +44,7 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { for _, h := range hashes { known[h] = struct{}{} } - removed, err := store.Sweep(func(hash string) bool { + removed, err := store.Sweep(cutoff, func(hash string) bool { _, ok := known[hash] return ok }) diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 2c8751674..0dd0b660f 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -60,6 +60,8 @@ var _ = Describe("Prune", func() { stray := []byte("no-row-bytes") h, _ := HashImage(bytes.NewReader(stray)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) Expect(Prune(context.Background(), ds, store)).To(Succeed()) diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index c87836ca9..145e0b3f7 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -45,16 +45,16 @@ func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) { } func (r *artworkRepository) PutImage(a *model.Artwork) error { - if a.CreatedAt.IsZero() { - a.CreatedAt = time.Now() - } + // created_at is the last-acquisition-write time the prune grace window keys on. + a.CreatedAt = time.Now() values, err := toSQLArgs(*a) if err != nil { return err } + // created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window. ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, - source_path=excluded.source_path, ref_mtime=excluded.ref_mtime`) + source_path=excluded.source_path, ref_mtime=excluded.ref_mtime, created_at=excluded.created_at`) _, err = r.executeSQL(ins) return err } diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index 0fdb122b6..12456bcd1 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -8,6 +8,7 @@ import ( "github.com/navidrome/navidrome/model" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pocketbase/dbx" ) // clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state. @@ -48,6 +49,19 @@ var _ = Describe("ArtworkRepository", func() { Expect(got.BlurHash).To(Equal("XYZ")) }) + It("refreshes created_at when reacquiring an existing hash", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed()) + _, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'"). + Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute() + Expect(err).ToNot(HaveOccurred()) + + Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed()) + + got, err := repo.GetImage("reacq") + Expect(err).ToNot(HaveOccurred()) + Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) + }) + It("returns ErrNotFound for a missing hash", func() { _, err := repo.GetImage("nope") Expect(err).To(MatchError(model.ErrNotFound)) diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go index dcec7486d..fceded1fb 100644 --- a/tests/mock_artwork_queue_repo.go +++ b/tests/mock_artwork_queue_repo.go @@ -11,6 +11,8 @@ type MockArtworkQueueRepo struct { model.ArtworkQueueRepository Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType) Err error + // ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state. + ItemArtworkSource *MockArtworkRepo } func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo { @@ -98,5 +100,28 @@ func (m *MockArtworkQueueRepo) Count() (int64, error) { } func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { - return 0, m.Err + if m.Err != nil || m.ItemArtworkSource == nil { + return 0, m.Err + } + now := time.Now() + var inserted int64 + for _, ia := range m.ItemArtworkSource.ItemData { + if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) { + continue + } + k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType) + if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows + continue + } + m.Data[k] = model.ArtworkQueueItem{ + ItemKind: ia.ItemKind, + ItemID: ia.ItemID, + ImageType: ia.ImageType, + Priority: model.ArtworkPriorityRecheck, + RetryAt: now, + EnqueuedAt: now, + } + inserted++ + } + return inserted, nil } diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index a6ca29df5..d8585fabf 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -267,7 +267,11 @@ func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRep if db.RealDS != nil { return db.RealDS.ArtworkQueue(ctx) } - db.MockedArtworkQueue = CreateMockArtworkQueueRepo() + q := CreateMockArtworkQueueRepo() + if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok { + q.ItemArtworkSource = aw + } + db.MockedArtworkQueue = q return db.MockedArtworkQueue } From 623b7d6a6c864f6c340db8fa7724001b0f0ddb36 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:11:15 -0400 Subject: [PATCH 14/51] fix(artwork): atomic orphan deletion and timestamp semantics from review DeleteOrphans re-checks age+references at delete time, PutItemArtwork defaults attempted_at, queue mock timestamps mirror SQL. --- core/artwork/prune.go | 29 ++++++++++++++++++-------- core/artwork/prune_test.go | 20 ++++++++++++++++++ model/artwork.go | 3 ++- persistence/artwork_repository.go | 13 ++++++++++-- persistence/artwork_repository_test.go | 27 ++++++++++++++++++++++-- tests/mock_artwork_queue_repo.go | 12 +++++------ tests/mock_artwork_repo.go | 18 +++++++++++++++- 7 files changed, 100 insertions(+), 22 deletions(-) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index b787dd1c9..de8811e14 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -16,24 +16,35 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { // One grace cutoff for both the DB orphan check and the file sweep: files younger // than the window may belong to acquisitions whose rows aren't committed yet. cutoff := time.Now().Add(-pruneMinAge) - orphans, err := repo.GetOrphanHashes(cutoff) + candidates, err := repo.GetOrphanHashes(cutoff) if err != nil { return err } - if len(orphans) > 0 { - arts, err := repo.GetImages(orphans) + if len(candidates) > 0 { + arts, err := repo.GetImages(candidates) if err != nil { return err } - if err := repo.DeleteImages(orphans...); err != nil { + if err := repo.DeleteOrphans(cutoff, candidates); err != nil { return err } - for _, a := range arts { - if err := store.Remove(a.Hash, a.Mime); err != nil { - log.Warn(ctx, "Prune: could not remove artwork file", "hash", a.Hash, err) - } + // DeleteOrphans may spare candidates reacquired since the snapshot; only remove files + // for rows actually gone (absent from the post-delete re-read). + survivors, err := repo.GetImages(candidates) + if err != nil { + return err } - log.Info(ctx, "Prune: removed orphan artwork", "count", len(orphans)) + removed := 0 + for _, h := range candidates { + if _, ok := survivors[h]; ok { + continue + } + if err := store.Remove(h, arts[h].Mime); err != nil { + log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err) + } + removed++ + } + log.Info(ctx, "Prune: removed orphan artwork", "count", removed) } hashes, err := repo.GetAllHashes() diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 0dd0b660f..aee8cd17c 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -56,6 +56,26 @@ var _ = Describe("Prune", func() { rc.Close() }) + It("spares a candidate reacquired between snapshot and delete", func() { + data := []byte("reacquired-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", + CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) + awRepo.OrphanHashes = []string{h} + // Reacquisition: an item now references the hash the snapshot flagged as orphan. + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", + ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).ToNot(HaveOccurred()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + It("sweeps store files that have no artwork row", func() { stray := []byte("no-row-bytes") h, _ := HashImage(bytes.NewReader(stray)) diff --git a/model/artwork.go b/model/artwork.go index 986a5c65d..5b50fbda0 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -65,7 +65,8 @@ type ArtworkRepository interface { GetImages(hashes []string) (map[string]Artwork, error) // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. GetOrphanHashes(createdBefore time.Time) ([]string, error) - DeleteImages(hashes ...string) error + // DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check). + DeleteOrphans(createdBefore time.Time, hashes []string) error // Per-item state (item_artwork table) GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error) PutItemArtwork(ia *ItemArtwork) error diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 145e0b3f7..a18b9d4af 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -92,9 +92,14 @@ func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, return hashes, err } -func (r *artworkRepository) DeleteImages(hashes ...string) error { +func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error { for chunk := range slices.Chunk(hashes, artworkBatchSize) { - if err := r.delete(Eq{"hash": chunk}); err != nil { + del := Delete(r.tableName).Where(And{ + Eq{"hash": chunk}, + Lt{"created_at": createdBefore}, + Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"), + }) + if _, err := r.executeSQL(del); err != nil { return err } } @@ -116,6 +121,10 @@ func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error { ia.ImageType = model.ImageTypePrimary } ia.UpdatedAt = time.Now() + // PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now. + if ia.AttemptedAt.IsZero() { + ia.AttemptedAt = ia.UpdatedAt + } values, err := toSQLArgs(*ia) if err != nil { return err diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index 12456bcd1..b3971f88f 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -99,11 +99,25 @@ var _ = Describe("ArtworkRepository", func() { Expect(orphans).To(BeEmpty()) }) - It("deletes by hashes", func() { + It("deletes only unreferenced hashes older than the cutoff", func() { Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) - Expect(repo.DeleteImages("d1")).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", + ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed()) + + Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed()) + _, err := repo.GetImage("d1") Expect(err).To(MatchError(model.ErrNotFound)) + _, err = repo.GetImage("dref") + Expect(err).ToNot(HaveOccurred()) + }) + + It("spares an unreferenced hash younger than the cutoff", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed()) + _, err := repo.GetImage("young") + Expect(err).ToNot(HaveOccurred()) }) It("fetches a batch larger than the SQL variable limit", func() { @@ -135,6 +149,15 @@ var _ = Describe("ArtworkRepository", func() { Expect(got.UpdatedAt).ToNot(BeZero()) }) + It("defaults attempted_at to now when unset", func() { + before := time.Now().Add(-time.Second) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt", + ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.AttemptedAt).To(BeTemporally(">", before)) + }) + It("represents known-absent as empty hash", func() { Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go index fceded1fb..f62afe43a 100644 --- a/tests/mock_artwork_queue_repo.go +++ b/tests/mock_artwork_queue_repo.go @@ -28,19 +28,17 @@ func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { if it.ImageType == "" { it.ImageType = model.ImageTypePrimary } - if it.RetryAt.IsZero() { - it.RetryAt = now - } k := iaKey(it.ItemKind, it.ItemID, it.ImageType) + // Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller. if prev, ok := m.Data[k]; ok { prev.Priority = max(prev.Priority, it.Priority) - prev.RetryAt = it.RetryAt + prev.RetryAt = now m.Data[k] = prev continue } - if it.EnqueuedAt.IsZero() { - it.EnqueuedAt = now - } + it.Attempts = 0 + it.RetryAt = now + it.EnqueuedAt = now m.Data[k] = it } return nil diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index 8ac4820e6..6575416b9 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -69,16 +69,29 @@ func (m *MockArtworkRepo) GetAllHashes() ([]string, error) { return hashes, nil } -func (m *MockArtworkRepo) DeleteImages(hashes ...string) error { +func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error { if m.Err != nil { return m.Err } + // Mirror the SQL reference re-check; createdBefore is ignored in the mock for simplicity. for _, h := range hashes { + if m.referenced(h) { + continue + } delete(m.Data, h) } return nil } +func (m *MockArtworkRepo) referenced(hash string) bool { + for _, ia := range m.ItemData { + if ia.Hash == hash { + return true + } + } + return false +} + func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { if m.Err != nil { return nil, m.Err @@ -97,6 +110,9 @@ func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error { ia.ImageType = model.ImageTypePrimary } ia.UpdatedAt = time.Now() + if ia.AttemptedAt.IsZero() { + ia.AttemptedAt = ia.UpdatedAt + } m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia return nil } From bf614e66adad3f1ce99a479fc1a8a1cd02e48d71 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:19:45 -0400 Subject: [PATCH 15/51] fix(artwork): guard orphan file removal with the prune grace window Duplicate ImageStore writes refresh the file mtime and Remove skips files newer than the cutoff, so overlapping acquisitions cannot lose their store files to a concurrent prune. --- core/artwork/image_store.go | 20 +++++++++++++++-- core/artwork/image_store_test.go | 38 +++++++++++++++++++++++++++++++- core/artwork/prune.go | 4 +++- core/artwork/prune_test.go | 19 ++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go index 517dfc879..ef27ed00a 100644 --- a/core/artwork/image_store.go +++ b/core/artwork/image_store.go @@ -54,6 +54,9 @@ func (s *ImageStore) path(hash, mimeType string) string { func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { dst := s.path(hash, mimeType) if _, err := os.Stat(dst); err == nil { + // A touched mtime marks the file live so a concurrent prune spares it. + now := time.Now() + _ = os.Chtimes(dst, now, now) return nil } if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { @@ -78,8 +81,21 @@ func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) { return os.Open(s.path(hash, mimeType)) } -func (s *ImageStore) Remove(hash, mimeType string) error { - err := os.Remove(s.path(hash, mimeType)) +// Remove deletes the store file unless it is newer than olderThan, in which case +// an overlapping acquisition may have just touched it and be about to commit its row. +func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error { + path := s.path(hash, mimeType) + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.ModTime().After(olderThan) { + return nil + } + err = os.Remove(path) if errors.Is(err, fs.ErrNotExist) { return nil } diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go index 2647f3b7f..89646deba 100644 --- a/core/artwork/image_store_test.go +++ b/core/artwork/image_store_test.go @@ -51,13 +51,49 @@ var _ = Describe("ImageStore", func() { Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) }) + It("refreshes the mtime on a duplicate write", func() { + data := []byte("touch-me") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + + info, err := os.Stat(store.path(h, "image/png")) + Expect(err).ToNot(HaveOccurred()) + Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute))) + }) + It("returns fs.ErrNotExist for missing images", func() { _, err := store.Open("beefbeefbeefbeef", "image/jpeg") Expect(os.IsNotExist(err)).To(BeTrue()) }) It("removes without error when already gone", func() { - Expect(store.Remove("beefbeefbeefbeef", "image/jpeg")).To(Succeed()) + Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed()) + }) + + It("spares a file newer than the cutoff, removes an aged one", func() { + fresh := []byte("fresh") + hf, _ := HashImage(bytes.NewReader(fresh)) + Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed()) + + aged := []byte("aged") + ha, _ := HashImage(bytes.NewReader(aged)) + Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed()) + + cutoff := time.Now().Add(-time.Hour) + Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed()) + Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed()) + + rc, err := store.Open(hf, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + _, err = store.Open(ha, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) }) It("sweeps unknown files, keeps known ones", func() { diff --git a/core/artwork/prune.go b/core/artwork/prune.go index de8811e14..70aac091e 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -39,7 +39,9 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { if _, ok := survivors[h]; ok { continue } - if err := store.Remove(h, arts[h].Mime); err != nil { + // A spared fresh file is at worst a stray a later sweep reclaims; full + // worker/prune mutual exclusion is Phase 2's concern. + if err := store.Remove(h, arts[h].Mime, cutoff); err != nil { log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err) } removed++ diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index aee8cd17c..9d4aad350 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -36,6 +36,8 @@ var _ = Describe("Prune", func() { data := []byte("orphan-bytes") h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) awRepo.OrphanHashes = []string{h} @@ -76,6 +78,23 @@ var _ = Describe("Prune", func() { rc.Close() }) + It("spares an orphan file freshly touched by an overlapping acquisition", func() { + data := []byte("racing-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", + CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) + awRepo.OrphanHashes = []string{h} + // The row is legitimately orphaned, but a concurrent acquisition just touched the + // file's mtime (duplicate Write) and is about to commit a row referencing it. + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + It("sweeps store files that have no artwork row", func() { stray := []byte("no-row-bytes") h, _ := HashImage(bytes.NewReader(stray)) From 8147f7c40bf2bafd58dc43c701dce5ad5e3908c3 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:28:51 -0400 Subject: [PATCH 16/51] fix(artwork): rewrite vanished duplicates and sweep stale mime variants Write falls through to a real write when the liveness touch fails, and sweep retention now matches the recorded mime's extension so obsolete variants are reclaimed. --- core/artwork/image_store.go | 11 ++++--- core/artwork/image_store_test.go | 44 ++++++++++++++++++++++++-- core/artwork/prune.go | 13 +++----- core/artwork/prune_test.go | 22 ++++++++++++- model/artwork.go | 4 +-- persistence/artwork_repository.go | 19 ++++++++--- persistence/artwork_repository_test.go | 7 ++-- tests/mock_artwork_repo.go | 10 +++--- 8 files changed, 99 insertions(+), 31 deletions(-) diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go index ef27ed00a..178c19d10 100644 --- a/core/artwork/image_store.go +++ b/core/artwork/image_store.go @@ -56,8 +56,10 @@ func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { if _, err := os.Stat(dst); err == nil { // A touched mtime marks the file live so a concurrent prune spares it. now := time.Now() - _ = os.Chtimes(dst, now, now) - return nil + if err := os.Chtimes(dst, now, now); err == nil { + return nil + } + // touch failed (file likely pruned concurrently) — fall through and write it } if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err @@ -104,7 +106,7 @@ func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error { // Sweep removes store files not accepted by keep. Files modified after cutoff // (including temp files) are always kept: their acquisition row may not be committed yet. -func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash string) bool) (int, error) { +func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) { removed := 0 err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { @@ -120,7 +122,8 @@ func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash string) bool) (int, name := d.Name() remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window if !remove { - remove = !keep(strings.TrimSuffix(name, filepath.Ext(name))) + ext := filepath.Ext(name) + remove = !keep(strings.TrimSuffix(name, ext), ext) } if remove { // #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go index 89646deba..0bf721623 100644 --- a/core/artwork/image_store_test.go +++ b/core/artwork/image_store_test.go @@ -65,6 +65,22 @@ var _ = Describe("ImageStore", func() { Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute))) }) + It("rewrites the bytes when the existing file vanished before the liveness touch", func() { + data := []byte("vanishing") + h, _ := HashImage(bytes.NewReader(data)) + for range 10 { + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(os.Remove(store.path(h, "image/png"))).To(Succeed()) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + + rc, err := store.Open(h, "image/png") + Expect(err).ToNot(HaveOccurred()) + got, _ := io.ReadAll(rc) + rc.Close() + Expect(got).To(Equal(data)) + } + }) + It("returns fs.ErrNotExist for missing images", func() { _, err := store.Open("beefbeefbeefbeef", "image/jpeg") Expect(os.IsNotExist(err)).To(BeTrue()) @@ -107,7 +123,7 @@ var _ = Describe("ImageStore", func() { old := time.Now().Add(-2 * time.Hour) Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed()) - removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h string) bool { return h == h1 }) + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 }) Expect(err).ToNot(HaveOccurred()) Expect(removed).To(Equal(1)) _, err = store.Open(h2, "image/jpeg") @@ -117,12 +133,34 @@ var _ = Describe("ImageStore", func() { rc.Close() }) + It("sweeps a stale mime variant of a known hash, keeps the current one", func() { + data := []byte("same-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + + // The recorded mime is image/jpeg, so the .png variant is obsolete. + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool { + return hash == h && ext == ".jpg" + }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h, "image/png") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + It("keeps young unknown files inside the grace window", func() { d := []byte("fresh-orphan") h, _ := HashImage(bytes.NewReader(d)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed()) - removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string) bool { return false }) + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false }) Expect(err).ToNot(HaveOccurred()) Expect(removed).To(Equal(0)) rc, err := store.Open(h, "image/jpeg") @@ -139,7 +177,7 @@ var _ = Describe("ImageStore", func() { freshTmp := filepath.Join(root, ".fresh.tmp") Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed()) - removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string) bool { return true }) + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true }) Expect(err).ToNot(HaveOccurred()) Expect(removed).To(Equal(1)) Expect(oldTmp).ToNot(BeAnExistingFile()) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index 70aac091e..db1e4fdc1 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -49,17 +49,14 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { log.Info(ctx, "Prune: removed orphan artwork", "count", removed) } - hashes, err := repo.GetAllHashes() + mimes, err := repo.GetAllMimes() if err != nil { return err } - known := make(map[string]struct{}, len(hashes)) - for _, h := range hashes { - known[h] = struct{}{} - } - removed, err := store.Sweep(cutoff, func(hash string) bool { - _, ok := known[hash] - return ok + removed, err := store.Sweep(cutoff, func(hash, ext string) bool { + // A known hash under a stale extension is a superseded mime variant — reclaim it. + m, ok := mimes[hash] + return ok && ext == extForMime(m) }) if err != nil { return err diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 9d4aad350..2e5c80d23 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -17,7 +17,7 @@ type flakyGetArtworkRepo struct { *tests.MockArtworkRepo } -func (f *flakyGetArtworkRepo) GetAllHashes() ([]string, error) { +func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) { return nil, errors.New("db locked") } @@ -108,6 +108,26 @@ var _ = Describe("Prune", func() { Expect(os.IsNotExist(err)).To(BeTrue()) }) + It("sweeps an obsolete mime variant of a reacquired hash", func() { + data := []byte("variant-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + // The row records the current mime; the .png file is a superseded variant. + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := store.Open(h, "image/png") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + It("never sweeps files on a transient DB error", func() { ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()} diff --git a/model/artwork.go b/model/artwork.go index 5b50fbda0..f5f3ab4db 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -73,8 +73,8 @@ type ArtworkRepository interface { DeleteForItem(kind, id string) error // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) - // GetAllHashes returns every stored artwork hash, for sweep membership checks. - GetAllHashes() ([]string, error) + // GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks. + GetAllMimes() (map[string]string, error) } type ArtworkQueueRepository interface { diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index a18b9d4af..05912a0d6 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -74,11 +74,20 @@ func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork return res, nil } -func (r *artworkRepository) GetAllHashes() ([]string, error) { - sel := Select("hash").From(r.tableName) - var hashes []string - err := r.queryAllSlice(sel, &hashes) - return hashes, err +func (r *artworkRepository) GetAllMimes() (map[string]string, error) { + sel := Select("hash", "mime").From(r.tableName) + var rows []struct { + Hash string + Mime string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + res := make(map[string]string, len(rows)) + for _, row := range rows { + res[row.Hash] = row.Mime + } + return res, nil } func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) { diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index b3971f88f..a13b1e7cb 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -76,12 +76,13 @@ var _ = Describe("ArtworkRepository", func() { Expect(got["b2"].Mime).To(Equal("image/png")) }) - It("returns every stored hash", func() { + It("returns every stored hash with its current mime", func() { Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed()) Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed()) - hashes, err := repo.GetAllHashes() + mimes, err := repo.GetAllMimes() Expect(err).ToNot(HaveOccurred()) - Expect(hashes).To(ConsistOf("all1", "all2")) + Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg")) + Expect(mimes).To(HaveKeyWithValue("all2", "image/png")) }) It("finds orphans older than cutoff, honoring item_artwork references", func() { diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index 6575416b9..851ee7f48 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -58,15 +58,15 @@ func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, er return m.OrphanHashes, m.Err } -func (m *MockArtworkRepo) GetAllHashes() ([]string, error) { +func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) { if m.Err != nil { return nil, m.Err } - hashes := make([]string, 0, len(m.Data)) - for h := range m.Data { - hashes = append(hashes, h) + mimes := make(map[string]string, len(m.Data)) + for h, a := range m.Data { + mimes[h] = a.Mime } - return hashes, nil + return mimes, nil } func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error { From 034cd17498cbee76b0044edfc9fd850407031630 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:36:22 -0400 Subject: [PATCH 17/51] fix(artwork): index artwork_queue in dequeue order The previous leading retry_at range column forced a temp B-tree sort of the whole eligible set on every DequeueBatch; ordering the index by (priority DESC, enqueued_at) lets scans stop after the batch size. --- db/migrations/20260722023032_add_artwork_tables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/migrations/20260722023032_add_artwork_tables.sql b/db/migrations/20260722023032_add_artwork_tables.sql index 94e0ebe5f..7133b71ce 100644 --- a/db/migrations/20260722023032_add_artwork_tables.sql +++ b/db/migrations/20260722023032_add_artwork_tables.sql @@ -33,7 +33,8 @@ CREATE TABLE artwork_queue ( enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (item_kind, item_id, image_type) ) WITHOUT ROWID; -CREATE INDEX ix_artwork_queue_drain ON artwork_queue(retry_at, priority, enqueued_at); +-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering. +CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at); -- +goose Down DROP TABLE artwork_queue; From 0147cc59b1a6dbefb0d688588b8d1c6d3f2bf066 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:42:42 -0400 Subject: [PATCH 18/51] fix(artwork): honor the orphan cutoff in the repository mock The mock's DeleteOrphans now applies createdBefore like the SQL implementation, and a new spec covers a freshly reacquired row surviving prune. --- core/artwork/prune_test.go | 18 ++++++++++++++++++ tests/mock_artwork_repo.go | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 2e5c80d23..4a45ddd13 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -78,6 +78,24 @@ var _ = Describe("Prune", func() { rc.Close() }) + It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() { + data := []byte("fresh-reacquired-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + // Reacquisition refreshed created_at after the snapshot; still unreferenced. + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", + CreatedAt: time.Now()})).To(Succeed()) + awRepo.OrphanHashes = []string{h} + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).ToNot(HaveOccurred()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + It("spares an orphan file freshly touched by an overlapping acquisition", func() { data := []byte("racing-bytes") h, _ := HashImage(bytes.NewReader(data)) diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index 851ee7f48..9565abd7b 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -73,11 +73,14 @@ func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string if m.Err != nil { return m.Err } - // Mirror the SQL reference re-check; createdBefore is ignored in the mock for simplicity. + // Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted. for _, h := range hashes { if m.referenced(h) { continue } + if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) { + continue + } delete(m.Data, h) } return nil From ebbe533c6a8c1876732d71385c2571673977ede5 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:51:13 -0400 Subject: [PATCH 19/51] fix(artwork): reject malformed hashes in ImageStore operations Known-absent states carry an empty hash and malformed persisted hashes could panic path sharding or inject separators; Write/Open/Remove now return an error for anything but 16 lowercase hex chars. --- core/artwork/image_store.go | 23 +++++++++++++++++++++++ core/artwork/image_store_test.go | 9 +++++++++ 2 files changed, 32 insertions(+) diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go index 178c19d10..4b4e482c9 100644 --- a/core/artwork/image_store.go +++ b/core/artwork/image_store.go @@ -47,11 +47,28 @@ func extForMime(m string) string { return ".img" } +// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "", +// and malformed persisted hashes must never reach path sharding (slice panics, separators). +func validHash(hash string) bool { + if len(hash) != 16 { + return false + } + for _, c := range []byte(hash) { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + func (s *ImageStore) path(hash, mimeType string) string { return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType)) } func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { + if !validHash(hash) { + return fmt.Errorf("imagestore: invalid hash %q", hash) + } dst := s.path(hash, mimeType) if _, err := os.Stat(dst); err == nil { // A touched mtime marks the file live so a concurrent prune spares it. @@ -80,12 +97,18 @@ func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { } func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) { + if !validHash(hash) { + return nil, fmt.Errorf("imagestore: invalid hash %q", hash) + } return os.Open(s.path(hash, mimeType)) } // Remove deletes the store file unless it is newer than olderThan, in which case // an overlapping acquisition may have just touched it and be about to commit its row. func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error { + if !validHash(hash) { + return fmt.Errorf("imagestore: invalid hash %q", hash) + } path := s.path(hash, mimeType) info, err := os.Stat(path) if errors.Is(err, fs.ErrNotExist) { diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go index 0bf721623..214cab3ad 100644 --- a/core/artwork/image_store_test.go +++ b/core/artwork/image_store_test.go @@ -90,6 +90,15 @@ var _ = Describe("ImageStore", func() { Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed()) }) + It("rejects invalid hashes instead of panicking", func() { + for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} { + Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash"))) + _, err := store.Open(h, "image/jpeg") + Expect(err).To(MatchError(ContainSubstring("invalid hash"))) + Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash"))) + } + }) + It("spares a file newer than the cutoff, removes an aged one", func() { fresh := []byte("fresh") hf, _ := HashImage(bytes.NewReader(fresh)) From c04c8ee02a68e7c97dc99f8b7f06066894aecf63 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 00:58:22 -0400 Subject: [PATCH 20/51] fix(artwork): mock PutImage refreshes created_at like the SQL repository Prune specs now age fixtures directly instead of seeding stale timestamps through the upsert. --- core/artwork/prune_test.go | 22 ++++++++++++++-------- tests/mock_artwork_repo.go | 5 ++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 4a45ddd13..1f621363b 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -32,14 +32,21 @@ var _ = Describe("Prune", func() { store = NewImageStore(GinkgoT().TempDir()) }) + // PutImage refreshes created_at like the SQL repo, so fixtures are aged directly. + ageArtwork := func(h string, t time.Time) { + a := awRepo.Data[h] + a.CreatedAt = t + awRepo.Data[h] = a + } + It("deletes orphan rows and their store files, keeps referenced ones", func() { data := []byte("orphan-bytes") h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) old := time.Now().Add(-2 * time.Hour) Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) - Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", - CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, old) awRepo.OrphanHashes = []string{h} kept := []byte("kept-bytes") @@ -62,8 +69,8 @@ var _ = Describe("Prune", func() { data := []byte("reacquired-bytes") h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) - Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", - CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, time.Now().Add(-2*time.Hour)) awRepo.OrphanHashes = []string{h} // Reacquisition: an item now references the hash the snapshot flagged as orphan. Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", @@ -83,8 +90,7 @@ var _ = Describe("Prune", func() { h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) // Reacquisition refreshed created_at after the snapshot; still unreferenced. - Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", - CreatedAt: time.Now()})).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) awRepo.OrphanHashes = []string{h} Expect(Prune(context.Background(), ds, store)).To(Succeed()) @@ -100,8 +106,8 @@ var _ = Describe("Prune", func() { data := []byte("racing-bytes") h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) - Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg", - CreatedAt: time.Now().Add(-2 * time.Hour)})).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, time.Now().Add(-2*time.Hour)) awRepo.OrphanHashes = []string{h} // The row is legitimately orphaned, but a concurrent acquisition just touched the // file's mtime (duplicate Write) and is about to commit a row referencing it. diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index 9565abd7b..13ad4d5e1 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -34,9 +34,8 @@ func (m *MockArtworkRepo) PutImage(a *model.Artwork) error { if m.Err != nil { return m.Err } - if a.CreatedAt.IsZero() { - a.CreatedAt = time.Now() - } + // Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly. + a.CreatedAt = time.Now() m.Data[a.Hash] = *a return nil } From 1f818e7633ae917e29b67004b18f894e93ac918d Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 15:52:54 -0400 Subject: [PATCH 21/51] fix(artwork): store backing-file provenance per item, not per hash --- .../20260722023032_add_artwork_tables.sql | 4 ++-- model/artwork.go | 20 ++++++++++--------- persistence/artwork_repository.go | 5 ++--- persistence/artwork_repository_test.go | 8 ++++++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/db/migrations/20260722023032_add_artwork_tables.sql b/db/migrations/20260722023032_add_artwork_tables.sql index 7133b71ce..bc400f90e 100644 --- a/db/migrations/20260722023032_add_artwork_tables.sql +++ b/db/migrations/20260722023032_add_artwork_tables.sql @@ -6,8 +6,6 @@ CREATE TABLE artwork ( height INTEGER NOT NULL DEFAULT 0, size_bytes INTEGER NOT NULL DEFAULT 0, blur_hash TEXT NOT NULL DEFAULT '', - source_path TEXT NOT NULL DEFAULT '', - ref_mtime INTEGER NOT NULL DEFAULT 0, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); @@ -17,6 +15,8 @@ CREATE TABLE item_artwork ( image_type TEXT NOT NULL DEFAULT 'primary', hash TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', + source_path TEXT NOT NULL DEFAULT '', + ref_mtime INTEGER NOT NULL DEFAULT 0, attempted_at TIMESTAMP, updated_at TIMESTAMP, PRIMARY KEY (item_kind, item_id, image_type) diff --git a/model/artwork.go b/model/artwork.go index f5f3ab4db..7cb117764 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -4,15 +4,13 @@ import "time" // Artwork is one unique image, identified by the XXH3-64 hash of its bytes. type Artwork struct { - Hash string `structs:"hash"` - Mime string `structs:"mime"` - Width int `structs:"width"` - Height int `structs:"height"` - SizeBytes int64 `structs:"size_bytes"` - BlurHash string `structs:"blur_hash"` - SourcePath string `structs:"source_path"` - RefMtime int64 `structs:"ref_mtime"` - CreatedAt time.Time `structs:"created_at"` + Hash string `structs:"hash"` + Mime string `structs:"mime"` + Width int `structs:"width"` + Height int `structs:"height"` + SizeBytes int64 `structs:"size_bytes"` + BlurHash string `structs:"blur_hash"` + CreatedAt time.Time `structs:"created_at"` } const ImageTypePrimary = "primary" @@ -24,6 +22,10 @@ type ItemArtwork struct { ImageType string `structs:"image_type"` Hash string `structs:"hash"` Source string `structs:"source"` + // SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise. + SourcePath string `structs:"source_path"` + // RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath. + RefMtime int64 `structs:"ref_mtime"` // attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork; // raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL. AttemptedAt time.Time `structs:"attempted_at"` diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 05912a0d6..505e3839d 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -53,8 +53,7 @@ func (r *artworkRepository) PutImage(a *model.Artwork) error { } // created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window. ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, - height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, - source_path=excluded.source_path, ref_mtime=excluded.ref_mtime, created_at=excluded.created_at`) + height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`) _, err = r.executeSQL(ins) return err } @@ -139,7 +138,7 @@ func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error { return err } ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET - hash=excluded.hash, source=excluded.source, + hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime, attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) _, err = r.items.executeSQL(ins) return err diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index a13b1e7cb..82c2888c8 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -137,16 +137,20 @@ var _ = Describe("ArtworkRepository", func() { }) Context("item state", func() { - It("upserts and reads state", func() { + It("upserts and reads state, including per-item provenance", func() { ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, - Hash: "h1", Source: "folder", AttemptedAt: time.Now()} + Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()} Expect(repo.PutItemArtwork(ia)).To(Succeed()) ia.Source = "embedded" + ia.SourcePath = "/music/a/track.mp3" + ia.RefMtime = 222 Expect(repo.PutItemArtwork(ia)).To(Succeed()) got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary) Expect(err).ToNot(HaveOccurred()) Expect(got.Source).To(Equal("embedded")) + Expect(got.SourcePath).To(Equal("/music/a/track.mp3")) + Expect(got.RefMtime).To(Equal(int64(222))) Expect(got.UpdatedAt).ToNot(BeZero()) }) From 2efa697e52433d106d1d2aa34e7c30de4168375a Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 08:22:49 -0400 Subject: [PATCH 22/51] feat(artwork): import blurhash encoder from #5797 --- core/artwork/blurhash/blurhash.go | 177 +++++++++++++++++++ core/artwork/blurhash/blurhash_bench_test.go | 41 +++++ core/artwork/blurhash/blurhash_suite_test.go | 17 ++ core/artwork/blurhash/blurhash_test.go | 113 ++++++++++++ 4 files changed, 348 insertions(+) create mode 100644 core/artwork/blurhash/blurhash.go create mode 100644 core/artwork/blurhash/blurhash_bench_test.go create mode 100644 core/artwork/blurhash/blurhash_suite_test.go create mode 100644 core/artwork/blurhash/blurhash_test.go diff --git a/core/artwork/blurhash/blurhash.go b/core/artwork/blurhash/blurhash.go new file mode 100644 index 000000000..302ec83c0 --- /dev/null +++ b/core/artwork/blurhash/blurhash.go @@ -0,0 +1,177 @@ +// Package blurhash implements the blurhash encoding algorithm (https://github.com/woltapp/blurhash), +// matching Jellyfin's parameters so clients tuned against Jellyfin see equivalent hashes. +package blurhash + +import ( + "errors" + "image" + "image/draw" + "math" + "strings" + "sync" + + xdraw "golang.org/x/image/draw" +) + +const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~" + +// maxInputSize matches Jellyfin: larger inputs are slower with no visually discernible difference. +const maxInputSize = 128 + +// Components picks x/y component counts for an image, targeting ~16 near-square tiles (Jellyfin's formula). +func Components(width, height int) (int, int) { + if width <= 0 || height <= 0 { + return 0, 0 + } + xf := math.Sqrt(16.0 * float64(width) / float64(height)) + yf := xf * float64(height) / float64(width) + return min(int(xf)+1, 9), min(int(yf)+1, 9) +} + +// Encode returns the blurhash of img using xComp x yComp components. +func Encode(img image.Image, xComp, yComp int) (string, error) { + if xComp < 1 || xComp > 9 || yComp < 1 || yComp > 9 { + return "", errors.New("blurhash: components must be between 1 and 9") + } + rgba := toRGBA(downscale(img)) + bounds := rgba.Bounds() + w, h := bounds.Dx(), bounds.Dy() + if w == 0 || h == 0 { + return "", errors.New("blurhash: empty image") + } + + cosX := make([][]float64, xComp) + for i := range cosX { + cosX[i] = make([]float64, w) + for x := range cosX[i] { + cosX[i][x] = math.Cos(math.Pi * float64(i) * float64(x) / float64(w)) + } + } + cosY := make([][]float64, yComp) + for j := range cosY { + cosY[j] = make([]float64, h) + for y := range cosY[j] { + cosY[j][y] = math.Cos(math.Pi * float64(j) * float64(y) / float64(h)) + } + } + + lin := srgbToLinearTable() + factors := make([][3]float64, xComp*yComp) + for y := 0; y < h; y++ { + row := rgba.Pix[y*rgba.Stride:] + for x := 0; x < w; x++ { + p := x * 4 + lr, lg, lb := lin[row[p]], lin[row[p+1]], lin[row[p+2]] + for j := 0; j < yComp; j++ { + for i := 0; i < xComp; i++ { + basis := cosX[i][x] * cosY[j][y] + f := &factors[j*xComp+i] + f[0] += basis * lr + f[1] += basis * lg + f[2] += basis * lb + } + } + } + } + for idx := range factors { + norm := 2.0 + if idx == 0 { + norm = 1.0 + } + scale := norm / float64(w*h) + factors[idx][0] *= scale + factors[idx][1] *= scale + factors[idx][2] *= scale + } + + var sb strings.Builder + sb.WriteString(Encode83((xComp-1)+(yComp-1)*9, 1)) + + ac := factors[1:] + maxVal := 1.0 + if len(ac) > 0 { + actualMax := 0.0 + for _, f := range ac { + actualMax = max(actualMax, math.Abs(f[0]), math.Abs(f[1]), math.Abs(f[2])) + } + quantMax := int(math.Max(0, math.Min(82, math.Floor(actualMax*166-0.5)))) + maxVal = float64(quantMax+1) / 166 + sb.WriteString(Encode83(quantMax, 1)) + } else { + sb.WriteString(Encode83(0, 1)) + } + + dc := factors[0] + sb.WriteString(Encode83(linearToSRGB(dc[0])<<16|linearToSRGB(dc[1])<<8|linearToSRGB(dc[2]), 4)) + for _, f := range ac { + sb.WriteString(Encode83(quantAC(f[0], maxVal)*19*19+quantAC(f[1], maxVal)*19+quantAC(f[2], maxVal), 2)) + } + return sb.String(), nil +} + +// toRGBA gives the pixel loop direct Pix access, avoiding a per-pixel allocation through the +// image.At interface (~16k allocs per encode). +func toRGBA(img image.Image) *image.RGBA { + if rgba, ok := img.(*image.RGBA); ok { + return rgba + } + b := img.Bounds() + dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) + draw.Draw(dst, dst.Bounds(), img, b.Min, draw.Src) + return dst +} + +var srgbToLinearTable = sync.OnceValue(func() *[256]float64 { + var t [256]float64 + for i := range t { + t[i] = srgbToLinear(i) + } + return &t +}) + +func downscale(img image.Image) image.Image { + b := img.Bounds() + w, h := b.Dx(), b.Dy() + if w <= maxInputSize && h <= maxInputSize { + return img + } + scale := float64(maxInputSize) / float64(max(w, h)) + dst := image.NewRGBA(image.Rect(0, 0, max(1, int(float64(w)*scale)), max(1, int(float64(h)*scale)))) + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), img, b, draw.Src, nil) + return dst +} + +func quantAC(v, maxVal float64) int { + return int(math.Max(0, math.Min(18, math.Floor(signPow(v/maxVal, 0.5)*9+9.5)))) +} + +func signPow(v, exp float64) float64 { + return math.Copysign(math.Pow(math.Abs(v), exp), v) +} + +func srgbToLinear(v int) float64 { + f := float64(v) / 255 + if f <= 0.04045 { + return f / 12.92 + } + return math.Pow((f+0.055)/1.055, 2.4) +} + +func linearToSRGB(v float64) int { + v = math.Min(math.Max(0, v), 1) + if v <= 0.0031308 { + return int(v*12.92*255 + 0.5) + } + return int((1.055*math.Pow(v, 1/2.4)-0.055)*255 + 0.5) +} + +// Encode83 encodes value as a fixed-width, big-endian base83 string of the given length, using the +// blurhash spec's alphabet. +func Encode83(value, length int) string { + b := make([]byte, length) + for i := length - 1; i >= 0; i-- { + b[i] = alphabet[value%83] + value /= 83 + } + return string(b) +} diff --git a/core/artwork/blurhash/blurhash_bench_test.go b/core/artwork/blurhash/blurhash_bench_test.go new file mode 100644 index 000000000..43b593eb5 --- /dev/null +++ b/core/artwork/blurhash/blurhash_bench_test.go @@ -0,0 +1,41 @@ +package blurhash_test + +import ( + "fmt" + "image" + "image/color" + "testing" + + "github.com/navidrome/navidrome/core/artwork/blurhash" +) + +// benchImage builds a deterministic gradient so runs are comparable across revisions. +func benchImage(size int) image.Image { + img := image.NewNRGBA(image.Rect(0, 0, size, size)) + for y := 0; y < size; y++ { + for x := 0; x < size; x++ { + img.SetNRGBA(x, y, color.NRGBA{ + R: uint8(255 * x / size), + G: uint8(255 * y / size), + B: uint8((x + y) * 255 / (2 * size)), + A: 255, + }) + } + } + return img +} + +func BenchmarkEncode(b *testing.B) { + for _, size := range []int{100, 300, 600, 900, 1200, 1500} { + img := benchImage(size) + x, y := blurhash.Components(size, size) + b.Run(fmt.Sprintf("%dx%d", size, size), func(b *testing.B) { + b.ReportAllocs() + for range b.N { + if _, err := blurhash.Encode(img, x, y); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/core/artwork/blurhash/blurhash_suite_test.go b/core/artwork/blurhash/blurhash_suite_test.go new file mode 100644 index 000000000..b0e96d7a7 --- /dev/null +++ b/core/artwork/blurhash/blurhash_suite_test.go @@ -0,0 +1,17 @@ +package blurhash_test + +import ( + "testing" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBlurHash(t *testing.T) { + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + RegisterFailHandler(Fail) + RunSpecs(t, "BlurHash Suite") +} diff --git a/core/artwork/blurhash/blurhash_test.go b/core/artwork/blurhash/blurhash_test.go new file mode 100644 index 000000000..18dd86b99 --- /dev/null +++ b/core/artwork/blurhash/blurhash_test.go @@ -0,0 +1,113 @@ +package blurhash_test + +import ( + "image" + "image/color" + "strings" + + "github.com/navidrome/navidrome/core/artwork/blurhash" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~" + +func decode83(s string) int { + v := 0 + for _, c := range s { + v = v*83 + strings.IndexRune(alphabet, c) + } + return v +} + +func solidImage(w, h int, c color.NRGBA) image.Image { + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.SetNRGBA(x, y, c) + } + } + return img +} + +func gradientImage(w, h int) image.Image { + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(255 * x / w), G: uint8(255 * y / h), B: 128, A: 255}) + } + } + return img +} + +var _ = Describe("Components", func() { + DescribeTable("derives component counts from aspect ratio (Jellyfin formula)", + func(w, h, expectedX, expectedY int) { + x, y := blurhash.Components(w, h) + Expect(x).To(Equal(expectedX)) + Expect(y).To(Equal(expectedY)) + }, + Entry("square album art", 600, 600, 5, 5), + Entry("small square", 1, 1, 5, 5), + Entry("landscape 16:9", 1920, 1080, 6, 4), + Entry("portrait 9:16", 1080, 1920, 4, 6), + Entry("extreme landscape capped at 9", 10000, 100, 9, 1), + Entry("zero width", 0, 600, 0, 0), + Entry("zero height", 600, 0, 0, 0), + ) +}) + +var _ = Describe("Encode", func() { + It("rejects out-of-range components", func() { + _, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 0, 5) + Expect(err).To(HaveOccurred()) + _, err = blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 5, 10) + Expect(err).To(HaveOccurred()) + }) + + It("produces the spec-mandated length", func() { + // 1 (size flag) + 1 (max AC) + 4 (DC) + 2 per AC component + h, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{R: 10, G: 20, B: 30, A: 255}), 4, 3) + Expect(err).ToNot(HaveOccurred()) + Expect(h).To(HaveLen(4 + 2 + 2*(4*3-1))) + }) + + It("encodes the size flag as the first character", func() { + h, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 4, 3) + Expect(err).ToNot(HaveOccurred()) + Expect(decode83(h[:1])).To(Equal((4 - 1) + (3-1)*9)) + }) + + It("stores the average color in the DC component", func() { + h, err := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 200, G: 100, B: 50, A: 255}), 4, 3) + Expect(err).ToNot(HaveOccurred()) + dc := decode83(h[2:6]) + Expect(dc >> 16).To(BeNumerically("~", 200, 1)) + Expect((dc >> 8) & 0xFF).To(BeNumerically("~", 100, 1)) + Expect(dc & 0xFF).To(BeNumerically("~", 50, 1)) + }) + + It("is deterministic", func() { + img := gradientImage(64, 64) + h1, err1 := blurhash.Encode(img, 5, 5) + h2, err2 := blurhash.Encode(img, 5, 5) + Expect(err1).ToNot(HaveOccurred()) + Expect(err2).ToNot(HaveOccurred()) + Expect(h1).To(Equal(h2)) + }) + + It("produces different hashes for different images", func() { + h1, _ := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 255, A: 255}), 4, 4) + h2, _ := blurhash.Encode(gradientImage(16, 16), 4, 4) + Expect(h1).ToNot(Equal(h2)) + }) + + It("downscales large images internally without changing the result materially", func() { + // A 1000px solid image must encode fine and carry the same DC as its small version. + big, err := blurhash.Encode(solidImage(1000, 1000, color.NRGBA{R: 60, G: 120, B: 180, A: 255}), 5, 5) + Expect(err).ToNot(HaveOccurred()) + small, err := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 60, G: 120, B: 180, A: 255}), 5, 5) + Expect(err).ToNot(HaveOccurred()) + Expect(big[2:6]).To(Equal(small[2:6])) + }) +}) From 967de74bf724eef5699b5806d793a7c9c48f8ea3 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 09:10:31 -0400 Subject: [PATCH 23/51] feat(artwork): add worker-side artwork resolvers --- core/artwork/resolve.go | 332 ++++++++++++++++++++++++++++++ core/artwork/resolve_test.go | 305 +++++++++++++++++++++++++++ tests/mock_playlist_track_repo.go | 8 + 3 files changed, 645 insertions(+) create mode 100644 core/artwork/resolve.go create mode 100644 core/artwork/resolve_test.go diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go new file mode 100644 index 000000000..bd314e37d --- /dev/null +++ b/core/artwork/resolve.go @@ -0,0 +1,332 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + "image/draw" + "image/png" + "io" + "io/fs" + "os" + "strings" + + "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/model" +) + +// resolution is one attempted acquisition outcome for an entity. +type resolution struct { + reader io.ReadCloser // nil when no source yielded an image + source string // model.ItemArtwork.Source value: "folder", "embedded", "external", "upload", "generated" + sourcePath string // backing library/upload file (folder/upload: the image; embedded: the audio file); "" otherwise + refMtime int64 // mtime of sourcePath at resolution time; 0 when no sourcePath + extError bool // an external source errored/timed out (forces failed, never absent) +} + +// extGateFunc is an alias for the external-step wrapper Task 4 injects (rate +// limiter + circuit breaker); resolveItem defaults to a plain passthrough. +type extGateFunc = func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) + +func passthroughExtGate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + return f() +} + +// resolveItem walks the kind's priority chain and returns the first hit. +func resolveItem(ctx context.Context, ds model.DataStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg, item model.ArtworkQueueItem, extGate func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error)) (resolution, error) { + if extGate == nil { + extGate = passthroughExtGate + } + switch item.ItemKind { + case "al": + return resolveAlbum(ctx, ds, prov, ffmpeg, item.ItemID, extGate) + case "ar": + return resolveArtist(ctx, ds, prov, ffmpeg, item.ItemID, extGate) + case "pl": + return resolvePlaylist(ctx, ds, prov, ffmpeg, item.ItemID, extGate) + case "ra": + return resolveRadio(ctx, ds, item.ItemID) + default: + return resolution{}, fmt.Errorf("resolveItem: kind %q is not resolvable by the worker", item.ItemKind) + } +} + +// resolveAlbum ports the folder/embedded/external selection from +// reader_album.go, walking conf.Server.CoverArtPriority. +func resolveAlbum(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, albumID string, extGate extGateFunc) (resolution, error) { + al, err := ds.Album(ctx).Get(albumID) + if err != nil { + return resolution{}, err + } + _, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, *al) + if err != nil { + return resolution{}, err + } + lib, err := loadLibraryView(ctx, ds, al.LibraryID) + if err != nil { + return resolution{}, err + } + + var extErr bool + for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.CoverArtPriority), ",") { + pattern = strings.TrimSpace(pattern) + switch { + case pattern == "embedded": + if res, ok := resolveEmbedded(ctx, lib, ffm, al.EmbedArtPath); ok { + return res, nil + } + case pattern == "external": + r, path, err := extGate(fromAlbumExternalSource(ctx, *al, prov)) + if r != nil { + return resolution{reader: r, source: "external", sourcePath: path}, nil + } + if err != nil && !errors.Is(err, model.ErrNotFound) { + extErr = true + } + case len(imgFiles) > 0: + if res, ok := resolveFolderFile(ctx, lib, imgFiles, pattern); ok { + return res, nil + } + } + } + return resolution{extError: extErr}, nil +} + +// resolveArtist ports the upload/folder/external selection from +// reader_artist.go: upload always wins, then conf.Server.ArtistArtPriority. +func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, artistID string, extGate extGateFunc) (resolution, error) { + ar, err := ds.Artist(ctx).Get(artistID) + if err != nil { + return resolution{}, err + } + if res, ok := resolveLocalFile(ar.UploadedImagePath(), "upload"); ok { + return res, nil + } + + // Only consider albums where the artist is the sole album artist, same as reader_artist.go. + als, err := ds.Album(ctx).GetAll(model.QueryOptions{ + Filters: squirrel.And{ + squirrel.Eq{"album_artist_id": artistID}, + squirrel.Eq{"json_array_length(participants, '$.albumartist')": 1}, + }, + }) + if err != nil { + return resolution{}, err + } + albumPaths, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, als...) + if err != nil { + return resolution{}, err + } + artistFolder, _, err := loadArtistFolder(ctx, ds, als, albumPaths) + if err != nil { + return resolution{}, err + } + var lib libraryView + if len(als) > 0 { + lib, err = loadLibraryView(ctx, ds, als[0].LibraryID) + if err != nil { + return resolution{}, err + } + } + + var extErr bool + for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.ArtistArtPriority), ",") { + pattern = strings.TrimSpace(pattern) + switch { + case pattern == "external": + r, path, err := extGate(fromArtistExternalSource(ctx, *ar, prov)) + if r != nil { + return resolution{reader: r, source: "external", sourcePath: path}, nil + } + if err != nil && !errors.Is(err, model.ErrNotFound) { + extErr = true + } + case pattern == "image-folder": + if res, ok := resolveArtistImageFolder(ar); ok { + return res, nil + } + case strings.HasPrefix(pattern, "album/"): + if lib.FS == nil { + continue + } + if res, ok := resolveFolderFile(ctx, lib, imgFiles, strings.TrimPrefix(pattern, "album/")); ok { + return res, nil + } + default: + if lib.FS == nil || artistFolder == "" { + continue + } + if res, ok := resolveArtistFolderPattern(ctx, lib, artistFolder, pattern); ok { + return res, nil + } + } + } + return resolution{extError: extErr}, nil +} + +// resolvePlaylist ports the 2x2 generated grid from reader_playlist.go, +// sourcing tiles through resolveAlbum instead of the old cached reader. +func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, playlistID string, extGate extGateFunc) (resolution, error) { + pl, err := ds.Playlist(ctx).Get(playlistID) + if err != nil { + return resolution{}, err + } + albumIDs, err := ds.Playlist(ctx).Tracks(pl.ID, false).GetAlbumIDs(model.QueryOptions{Max: 4, Sort: "random()"}) + if err != nil { + return resolution{}, err + } + + var tiles []image.Image + var extErr bool + for _, albumID := range albumIDs { + res, err := resolveAlbum(ctx, ds, prov, ffm, albumID, extGate) + if err != nil { + continue + } + if res.extError { + extErr = true + } + if res.reader == nil { + continue + } + tile, decErr := decodeTile(res.reader) + res.reader.Close() + if decErr == nil { + tiles = append(tiles, tile) + } + if len(tiles) == 4 { + break + } + } + // Grow to 4 tiles by repeating what we have, mirroring reader_playlist.go's loadTiles. + switch len(tiles) { + case 0: + return resolution{extError: extErr}, nil + case 2: + tiles = append(tiles, tiles[1], tiles[0]) + case 3: + tiles = append(tiles, tiles[0]) + } + r, err := assembleTiles(tiles) + if err != nil { + return resolution{extError: extErr}, nil //nolint:nilerr // encode failure is a soft "no image", not a resolveItem error + } + return resolution{reader: r, source: "generated"}, nil +} + +// resolveRadio ports reader_radio.go: only an uploaded image, no fallback. +func resolveRadio(ctx context.Context, ds model.DataStore, radioID string) (resolution, error) { + r, err := ds.Radio(ctx).Get(radioID) + if err != nil { + return resolution{}, err + } + res, _ := resolveLocalFile(r.UploadedImagePath(), "upload") + return res, nil +} + +func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, embedRel string) (resolution, bool) { + if embedRel == "" { + return resolution{}, false + } + abs := lib.Abs(embedRel) + mtime := mtimeViaFS(lib.FS, embedRel) + if r, _, _ := fromTag(ctx, lib.FS, embedRel)(); r != nil { + return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtime}, true + } + if r, _, _ := fromFFmpegTag(ctx, ffm, abs)(); r != nil { + return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtime}, true + } + return resolution{}, false +} + +func resolveFolderFile(ctx context.Context, lib libraryView, imgFiles []string, pattern string) (resolution, bool) { + r, path, _ := fromExternalFile(ctx, lib.FS, imgFiles, pattern)() + if r == nil { + return resolution{}, false + } + return resolution{reader: r, source: "folder", sourcePath: lib.Abs(path), refMtime: mtimeViaFS(lib.FS, path)}, true +} + +func resolveArtistImageFolder(ar *model.Artist) (resolution, bool) { + folder := conf.Server.ArtistImageFolder + if folder == "" { + return resolution{}, false + } + return resolveLocalFile(findImageInArtistFolder(folder, ar.MbzArtistID, ar.Name), "folder") +} + +func resolveArtistFolderPattern(ctx context.Context, lib libraryView, artistFolder, pattern string) (resolution, bool) { + r, path, _ := fromArtistFolder(ctx, lib.FS, lib.absRoot, artistFolder, pattern)() + if r == nil { + return resolution{}, false + } + return resolution{reader: r, source: "folder", sourcePath: path, refMtime: mtimeOf(path)}, true +} + +// resolveLocalFile opens an absolute path directly (uploads, image-folder). A +// missing or unreadable path is "no source", not an error. +func resolveLocalFile(path, source string) (resolution, bool) { + if path == "" { + return resolution{}, false + } + f, err := os.Open(path) + if err != nil { + return resolution{}, false + } + return resolution{reader: f, source: source, sourcePath: path, refMtime: mtimeOf(path)}, true +} + +func mtimeOf(path string) int64 { + info, err := os.Stat(path) + if err != nil { + return 0 + } + return info.ModTime().Unix() +} + +// mtimeViaFS stats through the library FS instead of a joined absolute path, +// since library roots in tests may not be real OS paths (e.g. testfile://). +func mtimeViaFS(fsys fs.FS, name string) int64 { + if fsys == nil || name == "" { + return 0 + } + info, err := fs.Stat(fsys, name) + if err != nil { + return 0 + } + return info.ModTime().Unix() +} + +// decodeTile and assembleTiles mirror playlistArtworkReader's createTile/ +// createTiledImage, reusing the same rect/fillCenter cropping helpers. +func decodeTile(r io.ReadCloser) (image.Image, error) { + img, _, err := image.Decode(r) + if err != nil { + return nil, err + } + return fillCenter(img, tileSize/2, tileSize/2), nil +} + +func assembleTiles(tiles []image.Image) (io.ReadCloser, error) { + buf := new(bytes.Buffer) + var err error + if len(tiles) == 4 { + rgba := image.NewRGBA(image.Rectangle{Max: image.Point{X: tileSize - 1, Y: tileSize - 1}}) + draw.Draw(rgba, rect(0), tiles[0], image.Point{}, draw.Src) + draw.Draw(rgba, rect(1), tiles[1], image.Point{}, draw.Src) + draw.Draw(rgba, rect(2), tiles[2], image.Point{}, draw.Src) + draw.Draw(rgba, rect(3), tiles[3], image.Point{}, draw.Src) + err = png.Encode(buf, rgba) + } else { + err = png.Encode(buf, tiles[0]) + } + if err != nil { + return nil, err + } + return io.NopCloser(buf), nil +} diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go new file mode 100644 index 000000000..c3fda1f1d --- /dev/null +++ b/core/artwork/resolve_test.go @@ -0,0 +1,305 @@ +package artwork + +import ( + "context" + "errors" + "image" + "io" + "net/url" + "os" + "path/filepath" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// fakeExternalProvider is a minimal external.Provider stub for resolve_test.go; +// only AlbumImage/ArtistImage are exercised by the resolvers. +type fakeExternalProvider struct { + external.Provider + albumImage func(ctx context.Context, id string) (*url.URL, error) + artistImage func(ctx context.Context, id string) (*url.URL, error) +} + +func (f *fakeExternalProvider) AlbumImage(ctx context.Context, id string) (*url.URL, error) { + if f.albumImage != nil { + return f.albumImage(ctx, id) + } + return nil, model.ErrNotFound +} + +func (f *fakeExternalProvider) ArtistImage(ctx context.Context, id string) (*url.URL, error) { + if f.artistImage != nil { + return f.artistImage(ctx, id) + } + return nil, model.ErrNotFound +} + +var _ = Describe("resolveItem", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + folderRepo *fakeFolderRepo + libRepo *tests.MockLibraryRepo + ffm *tests.MockFFmpeg + prov *fakeExternalProvider + repoRoot string + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + folderRepo = &fakeFolderRepo{} + libRepo = &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + ffm = tests.NewMockFFmpeg("") + prov = &fakeExternalProvider{} + ds = &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + } + }) + + Describe("kind dispatch", func() { + It("returns an error for kinds the worker never enqueues", func() { + _, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "mf", ItemID: "x"}, nil) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("album", func() { + BeforeEach(func() { + conf.Server.CoverArtPriority = "cover.jpg, embedded" + ds.MockedAlbum = tests.CreateMockAlbumRepo() + }) + + It("resolves folder art from the library FS", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}, + }) + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg")) + Expect(res.refMtime).To(BeNumerically(">", 0)) + Expect(res.extError).To(BeFalse()) + }) + + It("falls back to embedded art when no folder image matches", func() { + folderRepo.result = nil + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("embedded")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3")) + Expect(res.refMtime).To(BeNumerically(">", 0)) + }) + + It("sets extError when the external source errors without being not-found", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al3", Name: "Album"}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + }) + + It("does not set extError when the external source reports not-found", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al4", Name: "Album"}, + }) + // prov.albumImage left nil -> fakeExternalProvider returns model.ErrNotFound + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeFalse()) + }) + + It("routes the external step through a custom extGate", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al5", Name: "Album"}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("boom") + } + var extGateCalls int + extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + extGateCalls++ + return f() + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"}, extGate) + Expect(err).ToNot(HaveOccurred()) + Expect(res.extError).To(BeTrue()) + Expect(extGateCalls).To(Equal(1)) + }) + }) + + Describe("artist", func() { + It("resolves the uploaded image before any priority chain lookup", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "artist"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "artist", "ar1_test.jpg") + Expect(os.WriteFile(imgPath, []byte("uploaded artist image"), 0600)).To(Succeed()) + + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar1", Name: "Artist", UploadedImage: "ar1_test.jpg"}}) + ds.MockedArtist = artistRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("upload")) + Expect(res.sourcePath).To(Equal(imgPath)) + }) + + It("falls through to the ArtistArtPriority chain when there is no upload", func() { + conf.Server.ArtistArtPriority = "album/artist.*" + folderRepo.result = []model.Folder{{ + LibraryPath: testFileLibPath(repoRoot), + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"artist.png"}, + }} + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar2", Name: "Artist"}}) + ds.MockedArtist = artistRepo + ds.MockedAlbum = tests.CreateMockAlbumRepo() + ds.MockedAlbum.(*tests.MockAlbumRepo).All = model.Albums{ + {ID: "al9", Name: "Album", LibraryID: 0, FolderIDs: []string{"f1"}}, + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/artist.png")) + }) + }) + + Describe("radio", func() { + It("yields an empty resolution when there is no uploaded image", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio"}} + ds.MockedRadio = radioRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal(resolution{})) + }) + + It("resolves the uploaded image when set", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra2_test.jpg") + Expect(os.WriteFile(imgPath, []byte("uploaded radio image"), 0600)).To(Succeed()) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"ra2": {ID: "ra2", Name: "Radio", UploadedImage: "ra2_test.jpg"}} + ds.MockedRadio = radioRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("upload")) + Expect(res.sourcePath).To(Equal(imgPath)) + }) + }) + + Describe("playlist", func() { + BeforeEach(func() { + conf.Server.CoverArtPriority = "cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum = tests.CreateMockAlbumRepo() + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "t1", Name: "T1", FolderIDs: []string{"f1"}}, + {ID: "t2", Name: "T2", FolderIDs: []string{"f1"}}, + {ID: "t3", Name: "T3", FolderIDs: []string{"f1"}}, + {ID: "t4", Name: "T4", FolderIDs: []string{"f1"}}, + }) + }) + + DescribeTable("yields a generated grid from up to 4 album tiles", + func(albumIDs []string, expectedSize int) { + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl1", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: albumIDs} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl1"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("generated")) + + img, format, err := image.Decode(res.reader) + Expect(err).ToNot(HaveOccurred()) + Expect(format).To(Equal("png")) + Expect(img.Bounds().Dx()).To(Equal(expectedSize)) + Expect(img.Bounds().Dy()).To(Equal(expectedSize)) + }, + // tileSize-1: the 4-tile canvas is built as [0, tileSize-1], matching + // reader_playlist.go's createTiledImage exactly. + Entry("1 album -> single tile", []string{"t1"}, tileSize/2), + Entry("2 albums -> duplicated to 4 tiles", []string{"t1", "t2"}, tileSize-1), + Entry("3 albums -> duplicated to 4 tiles", []string{"t1", "t2", "t3"}, tileSize-1), + Entry("4 albums -> full grid", []string{"t1", "t2", "t3", "t4"}, tileSize-1), + ) + + It("yields an empty resolution when no album has art", func() { + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "empty1", Name: "Empty"}, + }) + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl2", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"empty1"}} + ds.MockedPlaylist = plRepo + folderRepo.result = nil + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl2"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.source).To(BeEmpty()) + }) + }) +}) diff --git a/tests/mock_playlist_track_repo.go b/tests/mock_playlist_track_repo.go index 2835baadd..83751bb28 100644 --- a/tests/mock_playlist_track_repo.go +++ b/tests/mock_playlist_track_repo.go @@ -14,6 +14,7 @@ type MockPlaylistTrackRepo struct { Reordered bool AddCount int Err error + AlbumIDs []string // stubbed result for GetAlbumIDs, ignoring options } func (m *MockPlaylistTrackRepo) SetData(tracks model.PlaylistTracks) { @@ -66,6 +67,13 @@ func (m *MockPlaylistTrackRepo) GetCursor(options ...model.QueryOptions) (model. }, nil } +func (m *MockPlaylistTrackRepo) GetAlbumIDs(...model.QueryOptions) ([]string, error) { + if m.Err != nil { + return nil, m.Err + } + return m.AlbumIDs, nil +} + func (m *MockPlaylistTrackRepo) GetMediaFileIDs(options ...model.QueryOptions) ([]string, error) { if m.Err != nil { return nil, m.Err From 608db503a747ce48044562699b46c56637b22322 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 09:22:08 -0400 Subject: [PATCH 24/51] fix(artwork): propagate playlist tile failures and dedupe external step --- core/artwork/resolve.go | 41 +++++++++++++++++------- core/artwork/resolve_test.go | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index bd314e37d..5aafdf5c5 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -81,11 +81,9 @@ func resolveAlbum(ctx context.Context, ds model.DataStore, prov external.Provide return res, nil } case pattern == "external": - r, path, err := extGate(fromAlbumExternalSource(ctx, *al, prov)) - if r != nil { - return resolution{reader: r, source: "external", sourcePath: path}, nil - } - if err != nil && !errors.Is(err, model.ErrNotFound) { + if res, ok, isErr := resolveExternalStep(extGate, fromAlbumExternalSource(ctx, *al, prov)); ok { + return res, nil + } else if isErr { extErr = true } case len(imgFiles) > 0: @@ -139,11 +137,9 @@ func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provid pattern = strings.TrimSpace(pattern) switch { case pattern == "external": - r, path, err := extGate(fromArtistExternalSource(ctx, *ar, prov)) - if r != nil { - return resolution{reader: r, source: "external", sourcePath: path}, nil - } - if err != nil && !errors.Is(err, model.ErrNotFound) { + if res, ok, isErr := resolveExternalStep(extGate, fromArtistExternalSource(ctx, *ar, prov)); ok { + return res, nil + } else if isErr { extErr = true } case pattern == "image-folder": @@ -183,9 +179,13 @@ func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Prov var tiles []image.Image var extErr bool + var tileErr error // first internal (non-external) tile failure, e.g. album deleted mid-flight for _, albumID := range albumIDs { res, err := resolveAlbum(ctx, ds, prov, ffm, albumID, extGate) if err != nil { + if tileErr == nil { + tileErr = err + } continue } if res.extError { @@ -203,10 +203,16 @@ func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Prov break } } + if len(tiles) == 0 { + // A tile-level failure must never resolve as a clean absent: propagate + // internal errors, and force extError for external ones. + if tileErr != nil { + return resolution{}, fmt.Errorf("resolvePlaylist: sampled album art failed: %w", tileErr) + } + return resolution{extError: extErr}, nil + } // Grow to 4 tiles by repeating what we have, mirroring reader_playlist.go's loadTiles. switch len(tiles) { - case 0: - return resolution{extError: extErr}, nil case 2: tiles = append(tiles, tiles[1], tiles[0]) case 3: @@ -229,6 +235,17 @@ func resolveRadio(ctx context.Context, ds model.DataStore, radioID string) (reso return res, nil } +// resolveExternalStep runs an external sourceFunc through extGate, shared by +// resolveAlbum and resolveArtist. ok reports a hit; extErr reports a +// non-not-found error (a not-found is a definitive "no", not a failure). +func resolveExternalStep(extGate extGateFunc, sf func() (io.ReadCloser, string, error)) (res resolution, ok bool, extErr bool) { + r, path, err := extGate(sf) + if r != nil { + return resolution{reader: r, source: "external", sourcePath: path}, true, false + } + return resolution{}, false, err != nil && !errors.Is(err, model.ErrNotFound) +} + func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, embedRel string) (resolution, bool) { if embedRel == "" { return resolution{}, false diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index c3fda1f1d..c4ccc3733 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -207,6 +207,54 @@ var _ = Describe("resolveItem", func() { Expect(res.source).To(Equal("folder")) Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/artist.png")) }) + + It("sets extError when the external source errors without being not-found", func() { + conf.Server.ArtistArtPriority = "external" + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar3", Name: "Artist"}}) + ds.MockedArtist = artistRepo + prov.artistImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar3"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + }) + + It("does not set extError when the external source reports not-found", func() { + conf.Server.ArtistArtPriority = "external" + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar4", Name: "Artist"}}) + ds.MockedArtist = artistRepo + // prov.artistImage left nil -> fakeExternalProvider returns model.ErrNotFound + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar4"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeFalse()) + }) + + It("routes the external step through a custom extGate", func() { + conf.Server.ArtistArtPriority = "external" + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar5", Name: "Artist"}}) + ds.MockedArtist = artistRepo + prov.artistImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("boom") + } + var extGateCalls int + extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + extGateCalls++ + return f() + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar5"}, extGate) + Expect(err).ToNot(HaveOccurred()) + Expect(res.extError).To(BeTrue()) + Expect(extGateCalls).To(Equal(1)) + }) }) Describe("radio", func() { @@ -301,5 +349,18 @@ var _ = Describe("resolveItem", func() { Expect(res.reader).To(BeNil()) Expect(res.source).To(BeEmpty()) }) + + It("does not resolve as absent when every sampled album fails to resolve", func() { + // "missing1"/"missing2" are not in MockAlbumRepo's data, so resolveAlbum + // returns a genuine (non-external) error for every sampled tile. + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl3", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"missing1", "missing2"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl3"}, nil) + Expect(err).To(HaveOccurred()) + Expect(res).To(Equal(resolution{})) + }) }) }) From e0655dc88263318cdfe1e6d834c3fbedf94dd733 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 09:36:12 -0400 Subject: [PATCH 25/51] feat(artwork): add acquisition processor Resolves one queue item end to end: hash/dedup, decode + 128px thumbnail blurhash, place bytes (store vs source file), and persist found/absent/ failed state for the worker (Task 4) to act on. --- core/artwork/artwork_suite_test.go | 8 ++ core/artwork/processor.go | 210 +++++++++++++++++++++++++++++ core/artwork/processor_test.go | 206 ++++++++++++++++++++++++++++ 3 files changed, 424 insertions(+) create mode 100644 core/artwork/processor.go create mode 100644 core/artwork/processor_test.go diff --git a/core/artwork/artwork_suite_test.go b/core/artwork/artwork_suite_test.go index d42d7f3e4..1fd04627a 100644 --- a/core/artwork/artwork_suite_test.go +++ b/core/artwork/artwork_suite_test.go @@ -15,9 +15,17 @@ import ( "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/goleak" ) func TestArtwork(t *testing.T) { + // Only run goleak checks when the GOLEAK env var is set + if os.Getenv("GOLEAK") != "" { + defer goleak.VerifyNone(t, + goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"), + ) + } + tests.Init(t, false) log.SetLevel(log.LevelFatal) RegisterFailHandler(Fail) diff --git a/core/artwork/processor.go b/core/artwork/processor.go new file mode 100644 index 000000000..659f414c2 --- /dev/null +++ b/core/artwork/processor.go @@ -0,0 +1,210 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + "image/draw" + "io" + "time" + + "github.com/navidrome/navidrome/core/artwork/blurhash" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + xdraw "golang.org/x/image/draw" +) + +// outcome tells the worker what to do with the queue row: found/absent +// delete it, failed reschedules it via MarkFailed. +type outcome int + +const ( + outcomeFound outcome = iota + outcomeAbsent + outcomeFailed +) + +// thumbnailSize is the max dimension fed to blurhash, matching Jellyfin's own input cap. +const thumbnailSize = 128 + +// workerDeps are the collaborators processItem needs; extGate is nil outside +// tests, in which case resolveItem falls back to a plain passthrough. +type workerDeps struct { + ds model.DataStore + store *ImageStore + prov external.Provider + ffmpeg ffmpeg.FFmpeg + extGate func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) +} + +// processItem resolves one queue item end to end: find an image, hash/decode/ +// blurhash it, place its bytes, and persist the resulting state. +func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueItem) outcome { + repo := deps.ds.Artwork(ctx) + + res, err := resolveItem(ctx, deps.ds, deps.prov, deps.ffmpeg, item, deps.extGate) + if err != nil { + log.Warn(ctx, "artwork: could not resolve item", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if res.reader == nil { + if res.extError { + // An external source errored/timed out: never settle on absent, keep serving old state. + return outcomeFailed + } + return writeAbsent(ctx, repo, item) + } + defer res.reader.Close() + + data, err := io.ReadAll(res.reader) + if err != nil { + log.Warn(ctx, "artwork: failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + log.Debug(ctx, "artwork: read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "bytes", len(data)) + + hash, err := HashImage(bytes.NewReader(data)) + if err != nil { + log.Warn(ctx, "artwork: failed to hash image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + + art, err := repo.GetImage(hash) + switch { + case err == nil: + // Dedup hit: identical bytes already known, reuse dims/mime/blurhash. + case errors.Is(err, model.ErrNotFound): + art, err = decodeArtwork(ctx, hash, data) + if err != nil { + log.Warn(ctx, "artwork: failed to decode resolved image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + default: + log.Warn(ctx, "artwork: failed to look up image hash", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + art.SizeBytes = int64(len(data)) + + if err := placeBytes(deps.store, art, res, data); err != nil { + log.Warn(ctx, "artwork: failed to write image store", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if err := repo.PutImage(art); err != nil { + log.Warn(ctx, "artwork: failed to persist artwork image", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + if err := repo.PutItemArtwork(&model.ItemArtwork{ + ItemKind: item.ItemKind, + ItemID: item.ItemID, + ImageType: item.ImageType, + Hash: hash, + Source: res.source, + AttemptedAt: time.Now(), + }); err != nil { + log.Warn(ctx, "artwork: failed to persist item artwork state", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + return outcomeFound +} + +// writeAbsent records a known-absent state: every local/external source answered definitively "no". +func writeAbsent(ctx context.Context, repo model.ArtworkRepository, item model.ArtworkQueueItem) outcome { + err := repo.PutItemArtwork(&model.ItemArtwork{ + ItemKind: item.ItemKind, + ItemID: item.ItemID, + ImageType: item.ImageType, + AttemptedAt: time.Now(), + }) + if err != nil { + log.Warn(ctx, "artwork: failed to persist absent state", "kind", item.ItemKind, "id", item.ItemID, err) + return outcomeFailed + } + return outcomeAbsent +} + +// decodeArtwork builds a new Artwork row from raw bytes: dimensions, mime and a +// blurhash computed from a downscaled thumbnail. +func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwork, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("decode image config: %w", err) + } + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("decode image: %w", err) + } + + thumb := makeThumbnail(img, thumbnailSize) + xComp, yComp := blurhash.Components(thumb.Bounds().Dx(), thumb.Bounds().Dy()) + bh, err := blurhash.Encode(thumb, xComp, yComp) + if err != nil { + log.Warn(ctx, "artwork: blurhash encoding failed", "hash", hash, err) + bh = "" + } + + return &model.Artwork{ + Hash: hash, + Mime: mimeForFormat(format), + Width: cfg.Width, + Height: cfg.Height, + BlurHash: bh, + }, nil +} + +// makeThumbnail downscales img to fit within maxSize on its longest side, +// reusing reader_resized.go's fast-scale-type + CatmullRom approach. Images +// already within bounds are returned as-is (no upscaling). +func makeThumbnail(img image.Image, maxSize int) image.Image { + b := img.Bounds() + w, h := b.Dx(), b.Dy() + if w <= maxSize && h <= maxSize { + return toFastScaleType(img) + } + scale := float64(maxSize) / float64(max(w, h)) + dst := image.NewRGBA(image.Rect(0, 0, max(1, int(float64(w)*scale)), max(1, int(float64(h)*scale)))) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), toFastScaleType(img), b, draw.Src, nil) + return dst +} + +// isFileBacked reports whether a resolution's bytes already live in a library/upload +// file, so the acquisition must not duplicate them into the content-addressed store. +func isFileBacked(source string) bool { + return source == "folder" || source == "upload" +} + +// placeBytes fills in art's SourcePath/RefMtime and, for sources with no library file +// backing them, writes the bytes into the store (embedded keeps its audio file provenance). +func placeBytes(store *ImageStore, art *model.Artwork, res resolution, data []byte) error { + if isFileBacked(res.source) { + art.SourcePath = res.sourcePath + art.RefMtime = res.refMtime + return nil + } + art.SourcePath = "" + art.RefMtime = 0 + if res.source == "embedded" { + art.SourcePath = res.sourcePath + art.RefMtime = res.refMtime + } + return store.Write(art.Hash, art.Mime, bytes.NewReader(data)) +} + +// mimeForFormat maps an image.Decode format name to its MIME type; extForMime +// in image_store.go performs the inverse for content-addressed file paths. +func mimeForFormat(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + case "gif": + return "image/gif" + case "webp": + return "image/webp" + } + return "application/octet-stream" +} diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go new file mode 100644 index 000000000..a73cd4fca --- /dev/null +++ b/core/artwork/processor_test.go @@ -0,0 +1,206 @@ +package artwork + +import ( + "context" + "errors" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("processItem", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + folderRepo *fakeFolderRepo + libRepo *tests.MockLibraryRepo + ffm *tests.MockFFmpeg + prov *fakeExternalProvider + store *ImageStore + artRepo *tests.MockArtworkRepo + repoRoot string + deps *workerDeps + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + folderRepo = &fakeFolderRepo{} + libRepo = &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + ffm = tests.NewMockFFmpeg("") + prov = &fakeExternalProvider{} + artRepo = tests.CreateMockArtworkRepo() + ds = &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + } + ds.MockedAlbum = tests.CreateMockAlbumRepo() + store = NewImageStore(GinkgoT().TempDir()) + deps = &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm} + + conf.Server.CoverArtPriority = "cover.jpg, embedded" + }) + + It("found-folder: persists state from a folder image, writes no store file, keeps sourcePath/refMtime", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}, + }) + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"}) + Expect(out).To(Equal(outcomeFound)) + + ia, err := artRepo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).ToNot(BeEmpty()) + Expect(ia.Source).To(Equal("folder")) + + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(filepath.ToSlash(art.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg")) + Expect(art.RefMtime).To(BeNumerically(">", 0)) + + _, err = store.Open(ia.Hash, art.Mime) + Expect(os.IsNotExist(err)).To(BeTrue(), "folder-backed art must not be duplicated into the store") + }) + + It("found-embedded: writes a store file and computes a non-empty blurhash from a real fixture", func() { + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + folderRepo.result = nil + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"}) + Expect(out).To(Equal(outcomeFound)) + + ia, err := artRepo.GetItemArtwork("al", "al2", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("embedded")) + + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(art.BlurHash).ToNot(BeEmpty()) + Expect(filepath.ToSlash(art.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3")) + + rc, err := store.Open(ia.Hash, art.Mime) + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("absent: no local source and no external error persists a known-absent state", func() { + folderRepo.result = nil + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al3", Name: "Album"}, + }) + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"}) + Expect(out).To(Equal(outcomeAbsent)) + + ia, err := artRepo.GetItemArtwork("al", "al3", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).To(BeEmpty()) + Expect(ia.Source).To(BeEmpty()) + Expect(ia.AttemptedAt).To(BeTemporally("~", time.Now(), time.Second)) + }) + + It("failed-on-extError: leaves the item's state untouched", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al4", Name: "Album"}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork("al", "al4", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("dedup: a second item with identical bytes skips decode and reuses the artwork row", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al5", Name: "Album A", FolderIDs: []string{"f1"}}, + {ID: "al6", Name: "Album B", FolderIDs: []string{"f1"}}, + }) + + out1 := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"}) + Expect(out1).To(Equal(outcomeFound)) + ia1, err := artRepo.GetItemArtwork("al", "al5", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + + // Poison the stored blurhash: if the second item re-decodes instead of + // deduping on hash, this sentinel gets overwritten by a real computed value. + poisoned := artRepo.Data[ia1.Hash] + poisoned.BlurHash = "SENTINEL" + artRepo.Data[ia1.Hash] = poisoned + + out2 := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"}) + Expect(out2).To(Equal(outcomeFound)) + ia2, err := artRepo.GetItemArtwork("al", "al6", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia2.Hash).To(Equal(ia1.Hash)) + + reused, err := artRepo.GetImage(ia1.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(reused.BlurHash).To(Equal("SENTINEL")) + }) + + It("decode failure on found bytes: fails without writing state", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra1_test.jpg") + Expect(os.WriteFile(imgPath, []byte("not actually an image"), 0600)).To(Succeed()) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio", UploadedImage: "ra1_test.jpg"}} + ds.MockedRadio = radioRepo + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork("ra", "ra1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("store write failure: fails without writing state", func() { + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al7", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + folderRepo.result = nil + + // A store root that is a plain file makes every MkdirAll under it fail. + blockedRoot := filepath.Join(GinkgoT().TempDir(), "not-a-dir") + Expect(os.WriteFile(blockedRoot, []byte("x"), 0600)).To(Succeed()) + deps.store = NewImageStore(blockedRoot) + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork("al", "al7", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) +}) From d6fc829f841e87bfaa1bba1cc4bd5cde6532c307 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 09:45:54 -0400 Subject: [PATCH 26/51] style(artwork): tighten processor comments to budget --- core/artwork/processor.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 659f414c2..0c4d6c520 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -28,7 +28,7 @@ const ( outcomeFailed ) -// thumbnailSize is the max dimension fed to blurhash, matching Jellyfin's own input cap. +// thumbnailSize is the max dimension fed to blurhash. const thumbnailSize = 128 // workerDeps are the collaborators processItem needs; extGate is nil outside @@ -155,9 +155,8 @@ func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwor }, nil } -// makeThumbnail downscales img to fit within maxSize on its longest side, -// reusing reader_resized.go's fast-scale-type + CatmullRom approach. Images -// already within bounds are returned as-is (no upscaling). +// makeThumbnail downscales img to fit within maxSize on its longest side. +// Images within bounds are returned as-is (no upscaling). func makeThumbnail(img image.Image, maxSize int) image.Image { b := img.Bounds() w, h := b.Dx(), b.Dy() From 57c64e386a647b619afd2431ba35aad4b261b7e2 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 10:01:47 -0400 Subject: [PATCH 27/51] feat(artwork): add acquisition worker service --- cmd/root.go | 10 ++ cmd/wire_gen.go | 15 ++ cmd/wire_injectors.go | 6 + conf/configuration.go | 4 + core/artwork/image_store.go | 8 + core/artwork/prune.go | 4 +- core/artwork/wire_providers.go | 2 + core/artwork/worker.go | 250 +++++++++++++++++++++++++++++ core/artwork/worker_test.go | 188 ++++++++++++++++++++++ core/artwork/worker_timing_test.go | 95 +++++++++++ 10 files changed, 580 insertions(+), 2 deletions(-) create mode 100644 core/artwork/worker.go create mode 100644 core/artwork/worker_test.go create mode 100644 core/artwork/worker_timing_test.go diff --git a/cmd/root.go b/cmd/root.go index 9e2b38cd8..2b64ab850 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -88,6 +88,7 @@ func runNavidrome(ctx context.Context) { g.Go(startInsightsCollector(ctx)) g.Go(scheduleDBAnalyzer(ctx)) g.Go(startPluginManager(ctx)) + g.Go(startArtworkWorker(ctx)) g.Go(runInitialScan(ctx)) if conf.Server.Scanner.Enabled { g.Go(startScanWatcher(ctx)) @@ -344,6 +345,15 @@ func startPlaybackServer(ctx context.Context) func() error { } } +// startArtworkWorker starts the background artwork acquisition worker. It always +// runs; the queue is simply empty until something enqueues work into it. +func startArtworkWorker(ctx context.Context) func() error { + return func() error { + log.Info(ctx, "Starting artwork worker") + return CreateArtworkWorker().Run(ctx) + } +} + // startPluginManager starts the plugin manager, if configured. func startPluginManager(ctx context.Context) func() error { return func() error { diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index c4a797ef4..44c4cb46d 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -236,6 +236,21 @@ func GetPlaybackServer() playback.PlaybackServer { return playbackServer } +func CreateArtworkWorker() *artwork.Worker { + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) + imageStore := artwork.ProvideImageStore() + broker := events.GetBroker() + metricsMetrics := metrics.GetPrometheusInstance(dataStore) + manager := plugins.GetManager(dataStore, broker, metricsMetrics) + agentsAgents := agents.GetAgents(dataStore, manager) + matcherMatcher := matcher.New(dataStore) + provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher) + fFmpeg := ffmpeg.New() + worker := artwork.NewWorker(dataStore, imageStore, provider, fFmpeg) + return worker +} + func getPluginManager() *plugins.Manager { sqlDB := db.Db() dataStore := persistence.New(sqlDB) diff --git a/cmd/wire_injectors.go b/cmd/wire_injectors.go index 94faa5af3..a835e5c4c 100644 --- a/cmd/wire_injectors.go +++ b/cmd/wire_injectors.go @@ -136,6 +136,12 @@ func GetPlaybackServer() playback.PlaybackServer { )) } +func CreateArtworkWorker() *artwork.Worker { + panic(wire.Build( + allProviders, + )) +} + func getPluginManager() *plugins.Manager { panic(wire.Build( allProviders, diff --git a/conf/configuration.go b/conf/configuration.go index 83793bd43..57ee4e4bc 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -139,6 +139,8 @@ type configOptions struct { DevArtworkThrottleBacklogLimit int DevArtworkThrottleBacklogTimeout time.Duration DevArtworkThrottleBuffered bool + DevArtworkWorkerConcurrency int + DevArtworkExternalRPS int DevArtistInfoTimeToLive time.Duration DevAlbumInfoTimeToLive time.Duration DevExternalScanner bool @@ -900,6 +902,8 @@ func setViperDefaults() { viper.SetDefault("devartworkthrottlebackloglimit", consts.RequestThrottleBacklogLimit) viper.SetDefault("devartworkthrottlebacklogtimeout", consts.RequestThrottleBacklogTimeout) viper.SetDefault("devartworkthrottlebuffered", true) + viper.SetDefault("devartworkworkerconcurrency", 2) + viper.SetDefault("devartworkexternalrps", 2) viper.SetDefault("devartistinfotimetolive", consts.ArtistInfoTimeToLive) viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive) viper.SetDefault("devexternalscanner", true) diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go index 4b4e482c9..53662f686 100644 --- a/core/artwork/image_store.go +++ b/core/artwork/image_store.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" "github.com/zeebo/xxh3" ) @@ -31,6 +33,12 @@ func NewImageStore(rootDir string) *ImageStore { return &ImageStore{root: rootDir} } +// ProvideImageStore roots the store in its own subtree under the data folder, so +// Prune's recursive sweep never reaches the per-entity upload folders next to it. +func ProvideImageStore() *ImageStore { + return NewImageStore(filepath.Join(conf.Server.DataFolder.String(), consts.ArtworkFolder, "store")) +} + // extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into // content-addressed paths and re-derived on Open, so they must be stable across OSes. func extForMime(m string) string { diff --git a/core/artwork/prune.go b/core/artwork/prune.go index db1e4fdc1..a40194b8a 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -39,8 +39,8 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { if _, ok := survivors[h]; ok { continue } - // A spared fresh file is at worst a stray a later sweep reclaims; full - // worker/prune mutual exclusion is Phase 2's concern. + // A spared fresh file is at worst a stray a later sweep reclaims; + // Worker.RunPrune serializes prune against in-flight acquisitions. if err := store.Remove(h, arts[h].Mime, cutoff); err != nil { log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err) } diff --git a/core/artwork/wire_providers.go b/core/artwork/wire_providers.go index 63231b54a..868ea8f56 100644 --- a/core/artwork/wire_providers.go +++ b/core/artwork/wire_providers.go @@ -8,4 +8,6 @@ var Set = wire.NewSet( NewArtwork, GetImageCache, NewCacheWarmer, + NewWorker, + ProvideImageStore, ) diff --git a/core/artwork/worker.go b/core/artwork/worker.go new file mode 100644 index 000000000..1ebe15527 --- /dev/null +++ b/core/artwork/worker.go @@ -0,0 +1,250 @@ +package artwork + +import ( + "context" + "errors" + "io" + "math" + "math/rand/v2" + "sync" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/core/external" + "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "golang.org/x/time/rate" +) + +const ( + workerPollInterval = 5 * time.Second + backoffBase = 5 * time.Minute + backoffCap = 48 * time.Hour + breakerThreshold = 5 + breakerProbeAfter = time.Minute +) + +var errBreakerOpen = errors.New("artwork: external circuit breaker open") + +// Worker drains the artwork queue and runs each item through processItem. The +// external step is rate-limited and circuit-broken; prune is serialized against +// in-flight acquisitions via pruneMu (acquisitions RLock, prune Lock). +type Worker struct { + deps workerDeps + limiter *rate.Limiter + breaker *breaker + pruneMu sync.RWMutex + wake chan struct{} + runCtx context.Context + + mu sync.Mutex + inFlight map[string]struct{} +} + +func NewWorker(ds model.DataStore, store *ImageStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg) *Worker { + rps := conf.Server.DevArtworkExternalRPS + limit := rate.Inf + if rps > 0 { + limit = rate.Limit(rps) + } + w := &Worker{ + deps: workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffmpeg}, + limiter: rate.NewLimiter(limit, max(1, rps)), + breaker: newBreaker(), + wake: make(chan struct{}, 1), + runCtx: context.Background(), + inFlight: map[string]struct{}{}, + } + w.deps.extGate = w.gate + return w +} + +// Run blocks draining the queue until ctx is cancelled. It exits cleanly with no +// leaked goroutines: each drain waits for its batch before the loop can return. +func (w *Worker) Run(ctx context.Context) error { + w.runCtx = ctx + concurrency := max(1, conf.Server.DevArtworkWorkerConcurrency) + ticker := time.NewTicker(workerPollInterval) + defer ticker.Stop() + for { + n, err := w.drain(ctx, concurrency) + if err != nil && ctx.Err() == nil { + log.Warn(ctx, "artwork: worker drain failed", err) + } + if ctx.Err() != nil { + return nil + } + if n > 0 { + continue // keep draining while the queue has ready work + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + case <-w.wake: + } + } +} + +// Bump enqueues an item at the highest priority and wakes the drain loop. It is +// non-blocking: a wake already pending is enough. +func (w *Worker) Bump(kind, id string) { + item := model.ArtworkQueueItem{ + ItemKind: kind, + ItemID: id, + ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityBump, + } + if err := w.deps.ds.ArtworkQueue(context.Background()).Enqueue(item); err != nil { + log.Warn("artwork: could not bump queue item", "kind", kind, "id", id, err) + return + } + select { + case w.wake <- struct{}{}: + default: + } +} + +// RunPrune runs Prune under the worker's write lock, so no acquisition can place +// a file while orphans are being reclaimed. This is the only sanctioned prune path. +func (w *Worker) RunPrune(ctx context.Context) error { + w.pruneMu.Lock() + defer w.pruneMu.Unlock() + return Prune(ctx, w.deps.ds, w.deps.store) +} + +func (w *Worker) drain(ctx context.Context, concurrency int) (int, error) { + batch, err := w.deps.ds.ArtworkQueue(ctx).DequeueBatch(2 * concurrency) + if err != nil { + return 0, err + } + items := w.claim(batch) + if len(items) == 0 { + return 0, nil + } + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + for _, item := range items { + sem <- struct{}{} + wg.Add(1) + go func(it model.ArtworkQueueItem) { + defer wg.Done() + defer func() { <-sem }() + defer w.release(it) + w.process(ctx, it) + }(item) + } + wg.Wait() + return len(items), nil +} + +func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) { + if item.ImageType == "" { + item.ImageType = model.ImageTypePrimary + } + w.pruneMu.RLock() + out := processItem(ctx, &w.deps, item) + w.pruneMu.RUnlock() + + queue := w.deps.ds.ArtworkQueue(ctx) + switch out { + case outcomeFound, outcomeAbsent: + if err := queue.Delete(item.ItemKind, item.ItemID, item.ImageType); err != nil { + log.Warn(ctx, "artwork: could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err) + } + case outcomeFailed: + retryAt := time.Now().Add(backoff(item.Attempts)) + if err := queue.MarkFailed(item.ItemKind, item.ItemID, item.ImageType, retryAt); err != nil { + log.Warn(ctx, "artwork: could not reschedule failed queue item", "kind", item.ItemKind, "id", item.ItemID, err) + } + } +} + +// claim reserves items not already in flight, so a wake-triggered re-drain never +// double-processes an item still running from a previous cycle. +func (w *Worker) claim(batch []model.ArtworkQueueItem) []model.ArtworkQueueItem { + w.mu.Lock() + defer w.mu.Unlock() + var out []model.ArtworkQueueItem + for _, it := range batch { + k := queueKey(it) + if _, busy := w.inFlight[k]; busy { + continue + } + w.inFlight[k] = struct{}{} + out = append(out, it) + } + return out +} + +func (w *Worker) release(it model.ArtworkQueueItem) { + w.mu.Lock() + delete(w.inFlight, queueKey(it)) + w.mu.Unlock() +} + +func queueKey(it model.ArtworkQueueItem) string { + return it.ItemKind + "|" + it.ItemID + "|" + it.ImageType +} + +// gate wraps the external step with the rate limiter and circuit breaker, matching +// extGateFunc so it can be injected via workerDeps.extGate. +func (w *Worker) gate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + if !w.breaker.allow() { + return nil, "", errBreakerOpen + } + if err := w.limiter.Wait(w.runCtx); err != nil { + return nil, "", err + } + r, path, err := f() + w.breaker.record(err) + return r, path, err +} + +// backoffFor returns min(5m×4^n, 48h) scaled by (1+jitter), with jitter in [-0.2, 0.2]. +func backoffFor(attempts int, jitter float64) time.Duration { + d := math.Min(float64(backoffBase)*math.Pow(4, float64(attempts)), float64(backoffCap)) + return time.Duration(d * (1 + jitter)) +} + +func backoff(attempts int) time.Duration { + return backoffFor(attempts, rand.Float64()*0.4-0.2) //nolint:gosec // retry jitter, not security-sensitive +} + +// breaker opens after breakerThreshold consecutive external errors and admits a +// single probe once breakerProbeAfter has elapsed; a success re-closes it. +type breaker struct { + mu sync.Mutex + failures int + openedAt time.Time +} + +func newBreaker() *breaker { return &breaker{} } + +func (b *breaker) allow() bool { + b.mu.Lock() + defer b.mu.Unlock() + if b.failures < breakerThreshold { + return true + } + if time.Since(b.openedAt) >= breakerProbeAfter { + b.openedAt = time.Now() // start a fresh probe window so only one caller passes + return true + } + return false +} + +func (b *breaker) record(err error) { + b.mu.Lock() + defer b.mu.Unlock() + // A not-found is a definitive answer, not a fault; only real errors trip the breaker. + if err == nil || errors.Is(err, model.ErrNotFound) { + b.failures = 0 + return + } + b.failures++ + if b.failures == breakerThreshold { + b.openedAt = time.Now() + } +} diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go new file mode 100644 index 000000000..d41249fa2 --- /dev/null +++ b/core/artwork/worker_test.go @@ -0,0 +1,188 @@ +package artwork + +import ( + "context" + "errors" + "io" + "net/url" + "os" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func findQueued(q *tests.MockArtworkQueueRepo, kind, id string) *model.ArtworkQueueItem { + for _, it := range q.Data { + if it.ItemKind == kind && it.ItemID == id { + return &it + } + } + return nil +} + +var _ = Describe("Worker", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + folderRepo *fakeFolderRepo + libRepo *tests.MockLibraryRepo + ffm *tests.MockFFmpeg + prov *fakeExternalProvider + store *ImageStore + artRepo *tests.MockArtworkRepo + queueRepo *tests.MockArtworkQueueRepo + repoRoot string + w *Worker + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + folderRepo = &fakeFolderRepo{} + libRepo = &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + ffm = tests.NewMockFFmpeg("") + prov = &fakeExternalProvider{} + artRepo = tests.CreateMockArtworkRepo() + queueRepo = tests.CreateMockArtworkQueueRepo() + ds = &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + MockedArtworkQueue: queueRepo, + } + ds.MockedAlbum = tests.CreateMockAlbumRepo() + store = NewImageStore(GinkgoT().TempDir()) + conf.Server.CoverArtPriority = "cover.jpg, embedded" + conf.Server.DevArtworkExternalRPS = 1000 // keep the limiter out of the way of behavior tests + w = NewWorker(ds, store, prov, ffm) + }) + + Describe("drain", func() { + It("processes a seeded queue item and removes it from the queue", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}, + }) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ + ItemKind: "al", ItemID: "al1", Priority: model.ArtworkPriorityScan, + })).To(Succeed()) + + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + ia, err := artRepo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder")) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero(), "a found item must be deleted from the queue") + }) + + It("reschedules a failed item via MarkFailed with a backed-off retry_at", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al4", Name: "Album"}}) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})).To(Succeed()) + + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + it := findQueued(queueRepo, "al", "al4") + Expect(it).ToNot(BeNil()) + Expect(it.Attempts).To(Equal(1)) + Expect(it.RetryAt).To(BeTemporally(">", time.Now())) + + _, err = artRepo.GetItemArtwork("al", "al4", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound), "a timeout must never settle on absent") + }) + + It("returns zero when the queue is empty", func() { + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(BeZero()) + }) + }) + + Describe("Bump", func() { + It("enqueues at Bump priority and wakes the loop", func() { + w.Bump("al", "al9") + it := findQueued(queueRepo, "al", "al9") + Expect(it).ToNot(BeNil()) + Expect(it.Priority).To(Equal(model.ArtworkPriorityBump)) + }) + }) + + Describe("gate/breaker", func() { + It("opens after 5 consecutive external errors and short-circuits the step", func() { + var calls int + failing := func() (io.ReadCloser, string, error) { + calls++ + return nil, "", errors.New("boom") + } + for range 5 { + _, _, err := w.gate(failing) + Expect(err).To(HaveOccurred()) + } + Expect(calls).To(Equal(5)) + + _, _, err := w.gate(failing) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(5), "an open breaker must not call the external step") + }) + + It("resets the failure count on a successful call", func() { + failing := func() (io.ReadCloser, string, error) { return nil, "", errors.New("boom") } + ok := func() (io.ReadCloser, string, error) { return io.NopCloser(nil), "p", nil } + for range 4 { + _, _, _ = w.gate(failing) + } + _, _, err := w.gate(ok) + Expect(err).ToNot(HaveOccurred()) + + var calls int + counting := func() (io.ReadCloser, string, error) { + calls++ + return nil, "", errors.New("boom") + } + for range 5 { + _, _, _ = w.gate(counting) + } + Expect(calls).To(Equal(5), "the breaker should have re-closed after the success") + }) + }) + + Describe("RunPrune", func() { + It("runs a prune under the worker mutex", func() { + Expect(w.RunPrune(ctx)).To(Succeed()) + }) + }) + + Describe("Run", func() { + It("exits cleanly when the context is cancelled", func() { + runCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- w.Run(runCtx) }() + + cancel() + Eventually(done, time.Second).Should(Receive(BeNil())) + }) + }) +}) diff --git a/core/artwork/worker_timing_test.go b/core/artwork/worker_timing_test.go new file mode 100644 index 000000000..a721459c2 --- /dev/null +++ b/core/artwork/worker_timing_test.go @@ -0,0 +1,95 @@ +package artwork + +import ( + "context" + "errors" + "testing" + "testing/synctest" + "time" + + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/gomega" + "go.uber.org/goleak" +) + +func TestArtworkBackoffSchedule(t *testing.T) { + g := NewWithT(t) + for _, c := range []struct { + attempts int + want time.Duration + }{ + {0, 5 * time.Minute}, + {1, 20 * time.Minute}, + {2, 80 * time.Minute}, + {3, 320 * time.Minute}, + {4, 1280 * time.Minute}, + {5, 48 * time.Hour}, + {6, 48 * time.Hour}, + } { + g.Expect(backoffFor(c.attempts, 0)).To(Equal(c.want), "attempt %d", c.attempts) + } + + base := backoffFor(2, 0) + g.Expect(backoffFor(2, 0.2)).To(Equal(time.Duration(float64(base) * 1.2))) + g.Expect(backoffFor(2, -0.2)).To(Equal(time.Duration(float64(base) * 0.8))) + + lo := time.Duration(float64(320*time.Minute) * 0.8) + hi := time.Duration(float64(320*time.Minute) * 1.2) + for range 200 { + d := backoff(3) + g.Expect(d).To(BeNumerically(">=", lo)) + g.Expect(d).To(BeNumerically("<=", hi)) + } +} + +func TestArtworkBreakerHalfOpen(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + g := NewWithT(t) + b := newBreaker() + + for range breakerThreshold { + b.record(errors.New("boom")) + } + g.Expect(b.allow()).To(BeFalse(), "breaker opens after consecutive errors") + + time.Sleep(breakerProbeAfter - time.Nanosecond) + g.Expect(b.allow()).To(BeFalse(), "still open before the probe interval") + + time.Sleep(time.Nanosecond) + g.Expect(b.allow()).To(BeTrue(), "half-open: one probe is granted") + g.Expect(b.allow()).To(BeFalse(), "only a single probe per interval") + + b.record(errors.New("boom")) // probe fails -> stay open + time.Sleep(breakerProbeAfter) + g.Expect(b.allow()).To(BeTrue(), "another probe after the next interval") + + b.record(nil) // probe succeeds -> close + g.Expect(b.allow()).To(BeTrue(), "closed breaker admits freely") + g.Expect(b.allow()).To(BeTrue()) + }) +} + +func TestArtworkWorkerRunNoLeak(t *testing.T) { + ignore := goleak.IgnoreCurrent() + defer goleak.VerifyNone(t, ignore) + + t.Cleanup(configtest.SetupConfig()) + ds := &tests.MockDataStore{MockedArtworkQueue: tests.CreateMockArtworkQueueRepo()} + w := NewWorker(ds, NewImageStore(t.TempDir()), &fakeExternalProvider{}, tests.NewMockFFmpeg("")) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- w.Run(ctx) }() + + time.Sleep(20 * time.Millisecond) // let the loop settle on the idle select + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned error on cancel: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Run did not exit after context cancel") + } +} From 25a05fd017e0226a8560c92e687ff1c77c13e4ed Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 10:21:48 -0400 Subject: [PATCH 28/51] feat(artwork): enqueue artwork resolution from scan and CRUD paths --- core/artwork/prune.go | 9 +++++++ core/artwork/prune_test.go | 19 ++++++++++++++ model/artwork.go | 2 ++ persistence/artwork_repository.go | 24 +++++++++++++++++ persistence/artwork_repository_test.go | 36 ++++++++++++++++++++++++++ persistence/radio_repository.go | 12 ++++++++- persistence/radio_repository_test.go | 21 +++++++++++++++ scanner/phase_1_folders.go | 17 ++++++++++++ scanner/phase_4_playlists.go | 5 ++++ scanner/phase_4_playlists_test.go | 23 ++++++++++++++++ scanner/scanner_test.go | 23 ++++++++++++++++ tests/mock_artwork_repo.go | 21 +++++++++++++++ 12 files changed, 211 insertions(+), 1 deletion(-) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index a40194b8a..5278beaa4 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -13,6 +13,15 @@ const pruneMinAge = time.Hour func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { repo := ds.Artwork(ctx) + + purged, err := repo.PurgeDanglingItemArtwork() + if err != nil { + return err + } + if purged > 0 { + log.Info(ctx, "Prune: purged dangling item artwork state", "count", purged) + } + // One grace cutoff for both the DB orphan check and the file sweep: files younger // than the window may belong to acquisitions whose rows aren't committed yet. cutoff := time.Now().Add(-pruneMinAge) diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 1f621363b..983e7c555 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -39,6 +39,25 @@ var _ = Describe("Prune", func() { awRepo.Data[h] = a } + It("purges dangling item_artwork state for gone entities, summed across kinds", func() { + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "gone-album", ImageType: model.ImageTypePrimary})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "gone-artist", ImageType: model.ImageTypePrimary})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "live-artist", ImageType: model.ImageTypePrimary})).To(Succeed()) + awRepo.ExistingIDs = map[string]map[string]bool{ + "al": {}, + "ar": {"live-artist": true}, + } + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetItemArtwork("al", "gone-album", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = awRepo.GetItemArtwork("ar", "gone-artist", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = awRepo.GetItemArtwork("ar", "live-artist", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + }) + It("deletes orphan rows and their store files, keeps referenced ones", func() { data := []byte("orphan-bytes") h, _ := HashImage(bytes.NewReader(data)) diff --git a/model/artwork.go b/model/artwork.go index 7cb117764..73db00ca5 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -77,6 +77,8 @@ type ArtworkRepository interface { GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) // GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks. GetAllMimes() (map[string]string, error) + // PurgeDanglingItemArtwork removes state rows whose entity no longer exists. + PurgeDanglingItemArtwork() (int64, error) } type ArtworkQueueRepository interface { diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 505e3839d..622f47bfe 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -114,6 +114,30 @@ func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []stri return nil } +// danglingItemArtworkKinds maps item_kind prefixes to the table that owns the entity. +var danglingItemArtworkKinds = map[string]string{ + "al": "album", + "ar": "artist", + "pl": "playlist", + "ra": "radio", +} + +func (r *artworkRepository) PurgeDanglingItemArtwork() (int64, error) { + var total int64 + for kind, table := range danglingItemArtworkKinds { + del := Delete(itemArtworkTable).Where(And{ + Eq{"item_kind": kind}, + Expr("item_id NOT IN (SELECT id FROM " + table + ")"), + }) + c, err := r.items.executeSQL(del) + if err != nil { + return total, err + } + total += c + } + return total, nil +} + func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { sel := Select("*").From(itemArtworkTable). Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index 82c2888c8..ae6752def 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -136,6 +136,42 @@ var _ = Describe("ArtworkRepository", func() { }) }) + Context("dangling state cleanup", func() { + It("purges item_artwork rows per kind whose entity no longer exists, summing counts", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "keepAl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "no-such-album", ImageType: model.ImageTypePrimary, Hash: "danglingAl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: artistKraftwerk.ID, ImageType: model.ImageTypePrimary, Hash: "keepAr"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "no-such-artist", ImageType: model.ImageTypePrimary, Hash: "danglingAr"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: plsBest.ID, ImageType: model.ImageTypePrimary, Hash: "keepPl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "no-such-playlist", ImageType: model.ImageTypePrimary, Hash: "danglingPl"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ra", ItemID: radioWithHomePage.ID, ImageType: model.ImageTypePrimary, Hash: "keepRa"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ra", ItemID: "no-such-radio", ImageType: model.ImageTypePrimary, Hash: "danglingRa"})).To(Succeed()) + + purged, err := repo.PurgeDanglingItemArtwork() + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(Equal(int64(4))) + + for _, kept := range []model.ItemArtwork{ + {ItemKind: "al", ItemID: albumSgtPeppers.ID}, + {ItemKind: "ar", ItemID: artistKraftwerk.ID}, + {ItemKind: "pl", ItemID: plsBest.ID}, + {ItemKind: "ra", ItemID: radioWithHomePage.ID}, + } { + _, err := repo.GetItemArtwork(kept.ItemKind, kept.ItemID, model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + } + for _, gone := range []model.ItemArtwork{ + {ItemKind: "al", ItemID: "no-such-album"}, + {ItemKind: "ar", ItemID: "no-such-artist"}, + {ItemKind: "pl", ItemID: "no-such-playlist"}, + {ItemKind: "ra", ItemID: "no-such-radio"}, + } { + _, err := repo.GetItemArtwork(gone.ItemKind, gone.ItemID, model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + } + }) + }) + Context("item state", func() { It("upserts and reads state, including per-item provenance", func() { ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, diff --git a/persistence/radio_repository.go b/persistence/radio_repository.go index a073643db..20f82ee36 100644 --- a/persistence/radio_repository.go +++ b/persistence/radio_repository.go @@ -7,6 +7,7 @@ import ( . "github.com/Masterminds/squirrel" "github.com/deluan/rest" + "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/id" "github.com/pocketbase/dbx" @@ -72,7 +73,16 @@ func (r *radioRepository) Put(radio *model.Radio, colsToUpdate ...string) error colsToUpdate = append(colsToUpdate, "UpdatedAt") } _, err := r.put(radio.ID, radio, colsToUpdate...) - return err + if err != nil { + return err + } + // Enqueue artwork resolution for the created/updated radio. Never fails the save. + item := model.ArtworkQueueItem{ItemKind: "ra", ItemID: radio.ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan} + if err := NewArtworkQueueRepository(r.ctx, r.db).Enqueue(item); err != nil { + log.Warn(r.ctx, "could not enqueue radio artwork", "id", radio.ID, err) + } + return nil } func (r *radioRepository) Count(options ...rest.QueryOptions) (int64, error) { diff --git a/persistence/radio_repository_test.go b/persistence/radio_repository_test.go index 05628ca41..1d6cce4f8 100644 --- a/persistence/radio_repository_test.go +++ b/persistence/radio_repository_test.go @@ -107,6 +107,27 @@ var _ = Describe("RadioRepository", func() { Expect(err).To(BeNil()) Expect(all[2].StreamUrl).To(Equal("https://example.com:4533/app")) }) + + It("enqueues artwork resolution for the saved radio", func() { + err := repo.Put(&model.Radio{ + Name: "Artwork radio", + StreamUrl: "https://example.com:4533/artwork", + }) + Expect(err).To(BeNil()) + + all, err := repo.GetAll() + Expect(err).To(BeNil()) + created := all[len(all)-1] + + queueRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + queued, err := queueRepo.DequeueBatch(1000) + Expect(err).To(BeNil()) + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "ra"), + HaveField("ItemID", created.ID), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + }) }) }) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 5e898590b..3a7265cc4 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -332,6 +332,8 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) // Collect artwork IDs to pre-cache after the transaction commits var artworkIDs []model.ArtworkID + // Collect artwork queue items for changed albums/artists, enqueued in the same transaction + var queueItems []model.ArtworkQueueItem err := p.ds.WithTx(func(tx model.DataStore) error { // Instantiate all repositories just once per folder @@ -372,6 +374,10 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) } if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists { artworkIDs = append(artworkIDs, entry.artists[i].CoverArtID()) + queueItems = append(queueItems, model.ArtworkQueueItem{ + ItemKind: "ar", ItemID: entry.artists[i].ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan, + }) } } @@ -384,6 +390,10 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) } if entry.albums[i].Name != consts.UnknownAlbum { artworkIDs = append(artworkIDs, entry.albums[i].CoverArtID()) + queueItems = append(queueItems, model.ArtworkQueueItem{ + ItemKind: "al", ItemID: entry.albums[i].ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan, + }) } } @@ -415,6 +425,13 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) return err } } + + // Enqueue artwork resolution for changed albums/artists. Never fails the scan. + if len(queueItems) > 0 { + if err := tx.ArtworkQueue(p.ctx).Enqueue(queueItems...); err != nil { + log.Warn(p.ctx, "Scanner: could not enqueue artwork resolution", "folder", entry.path, err) + } + } return nil }, "scanner: persist changes") if err != nil { diff --git a/scanner/phase_4_playlists.go b/scanner/phase_4_playlists.go index 8ba014235..15798b054 100644 --- a/scanner/phase_4_playlists.go +++ b/scanner/phase_4_playlists.go @@ -149,6 +149,11 @@ func (p *phasePlaylists) processPlaylistsInFolder(folder *model.Folder) (*model. log.Debug("Scanner: Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks), "elapsed", time.Since(started)) } p.cw.PreCache(pls.CoverArtID()) + item := model.ArtworkQueueItem{ItemKind: "pl", ItemID: pls.ID, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan} + if err := p.ds.ArtworkQueue(p.ctx).Enqueue(item); err != nil { + log.Warn(p.ctx, "Scanner: could not enqueue playlist artwork", "id", pls.ID, err) + } p.refreshed.Add(1) } return folder, nil diff --git a/scanner/phase_4_playlists_test.go b/scanner/phase_4_playlists_test.go index 49ffc7fb7..1adf34a63 100644 --- a/scanner/phase_4_playlists_test.go +++ b/scanner/phase_4_playlists_test.go @@ -193,6 +193,29 @@ var _ = Describe("phasePlaylists", func() { Expect(phase.refreshed.Load()).To(Equal(uint32(2))) }) + It("enqueues artwork resolution for the imported playlist", func() { + libPath := GinkgoT().TempDir() + folder := &model.Folder{LibraryPath: libPath, Path: "path/to", Name: "folder"} + _ = os.MkdirAll(folder.AbsolutePath(), 0755) + + file1 := filepath.Join(folder.AbsolutePath(), "playlist1.m3u") + _ = os.WriteFile(file1, []byte{}, 0600) + + pls.On("ImportFromFolder", mock.Anything, folder, "playlist1.m3u"). + Return(&model.Playlist{ID: "pl1"}, nil) + + _, err := phase.processPlaylistsInFolder(folder) + Expect(err).ToNot(HaveOccurred()) + + queued, err := ds.ArtworkQueue(ctx).DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "pl"), + HaveField("ItemID", "pl1"), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + }) + It("reports an error if there is an error reading files", func() { tests.SkipOnWindows("relies on Unix /etc filesystem") progress := make(chan *ProgressInfo) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index cc3720717..0a7fb07b2 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -152,6 +152,29 @@ var _ = Describe("Scanner", Ordered, func() { HaveField("SongCount", Equal(4)), )) }) + It("should enqueue artwork resolution for the scanned albums and artists", func() { + Expect(runScanner(ctx, true)).To(Succeed()) + + albums, _ := ds.Album(ctx).GetAll() + artists, _ := ds.Artist(ctx).GetAll(model.QueryOptions{Filters: squirrel.NotEq{"name": consts.UnknownArtist}}) + queued, err := ds.ArtworkQueue(ctx).DequeueBatch(1000) + Expect(err).ToNot(HaveOccurred()) + + for _, al := range albums { + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "al"), + HaveField("ItemID", al.ID), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + } + for _, ar := range artists { + Expect(queued).To(ContainElement(SatisfyAll( + HaveField("ItemKind", "ar"), + HaveField("ItemID", ar.ID), + HaveField("Priority", model.ArtworkPriorityScan), + ))) + } + }) }) When("a file was changed", func() { It("should update the media_file", func() { diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index 13ad4d5e1..db73af84d 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -12,6 +12,8 @@ type MockArtworkRepo struct { ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType) OrphanHashes []string Err error + // ExistingIDs, keyed by item_kind, backs PurgeDanglingItemArtwork; nil map keeps everything. + ExistingIDs map[string]map[string]bool } func CreateMockArtworkRepo() *MockArtworkRepo { @@ -68,6 +70,25 @@ func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) { return mimes, nil } +func (m *MockArtworkRepo) PurgeDanglingItemArtwork() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + var purged int64 + for k, ia := range m.ItemData { + // A nil per-kind map means that kind isn't tracked by the test, so keep it. + existing := m.ExistingIDs[ia.ItemKind] + if existing == nil { + continue + } + if !existing[ia.ItemID] { + delete(m.ItemData, k) + purged++ + } + } + return purged, nil +} + func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error { if m.Err != nil { return m.Err From ad38cd1d584e83e347372cbf3a98ae3068b12a65 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 10:40:34 -0400 Subject: [PATCH 29/51] feat(artwork): artwork backfill, fingerprint re-resolution and scheduled jobs --- cmd/root.go | 54 +++++++++- consts/consts.go | 4 + core/artwork/housekeeping.go | 120 +++++++++++++++++++++ core/artwork/housekeeping_test.go | 174 ++++++++++++++++++++++++++++++ 4 files changed, 349 insertions(+), 3 deletions(-) create mode 100644 core/artwork/housekeeping.go create mode 100644 core/artwork/housekeeping_test.go diff --git a/cmd/root.go b/cmd/root.go index 2b64ab850..4df5c1cb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -88,7 +89,9 @@ func runNavidrome(ctx context.Context) { g.Go(startInsightsCollector(ctx)) g.Go(scheduleDBAnalyzer(ctx)) g.Go(startPluginManager(ctx)) - g.Go(startArtworkWorker(ctx)) + artworkWorker := CreateArtworkWorker() + g.Go(startArtworkWorker(ctx, artworkWorker)) + g.Go(scheduleArtworkHousekeeping(ctx, artworkWorker)) g.Go(runInitialScan(ctx)) if conf.Server.Scanner.Enabled { g.Go(startScanWatcher(ctx)) @@ -347,10 +350,55 @@ func startPlaybackServer(ctx context.Context) func() error { // startArtworkWorker starts the background artwork acquisition worker. It always // runs; the queue is simply empty until something enqueues work into it. -func startArtworkWorker(ctx context.Context) func() error { +func startArtworkWorker(ctx context.Context, worker *artwork.Worker) func() error { return func() error { log.Info(ctx, "Starting artwork worker") - return CreateArtworkWorker().Run(ctx) + return worker.Run(ctx) + } +} + +// scheduleArtworkHousekeeping runs the startup fingerprint backfill and registers the +// recurring stale-absent recheck and prune jobs. Scan-triggered prune lands in a later phase. +func scheduleArtworkHousekeeping(ctx context.Context, worker *artwork.Worker) func() error { + return func() error { + ds := CreateDataStore() + schedulerInstance := scheduler.GetInstance() + + if _, err := schedulerInstance.Add(consts.ArtworkStaleAbsentRecheckSchedule, func() { + if err := artwork.EnqueueStaleAbsentAll(ctx, ds); err != nil { + log.Error(ctx, "Error enqueueing stale artwork rechecks", err) + } + }); err != nil { + log.Error(ctx, "Error scheduling artwork stale-absent recheck", err) + } + + if _, err := schedulerInstance.Add(consts.ArtworkPruneSchedule, func() { + if err := worker.RunPrune(ctx); err != nil { + log.Error(ctx, "Error running artwork prune", err) + } + }); err != nil { + log.Error(ctx, "Error scheduling artwork prune", err) + } + + backfilled, err := artwork.Backfill(ctx, ds) + if err != nil { + log.Error(ctx, "Error running artwork backfill", err) + return nil + } + if !backfilled { + return nil + } + log.Info(ctx, "Artwork backfill enqueued, scheduling a follow-up prune") + timer := time.NewTimer(consts.ArtworkPostBackfillPruneDelay) + defer timer.Stop() + select { + case <-timer.C: + if err := worker.RunPrune(ctx); err != nil { + log.Error(ctx, "Error running post-backfill artwork prune", err) + } + case <-ctx.Done(): + } + return nil } } diff --git a/consts/consts.go b/consts/consts.go index f453ac125..0a244f55f 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -35,6 +35,10 @@ const ( DBAnalyzeCheckSchedule = "@every 30m" DBAnalyzeMaxAge = 24 * time.Hour + ArtworkStaleAbsentRecheckSchedule = "@every 1h" + ArtworkPruneSchedule = "@daily" + ArtworkPostBackfillPruneDelay = 10 * time.Minute + // DefaultEncryptionKey This is the encryption key used if none is specified in the `PasswordEncryptionKey` option // Never ever change this! Or it will break all Navidrome installations that don't set the config option DefaultEncryptionKey = "just for obfuscation" diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go new file mode 100644 index 000000000..6f1fa20a3 --- /dev/null +++ b/core/artwork/housekeeping.go @@ -0,0 +1,120 @@ +package artwork + +import ( + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against +// to detect artwork-affecting config changes across restarts. +const FingerprintPropertyKey = "artwork.fingerprint" + +// staleAbsentAge is how old an absent resolution must be before the recheck job retries it. +const staleAbsentAge = 24 * time.Hour + +// staleAbsentKinds are the item kinds eligible for the periodic stale-absent recheck. +var staleAbsentKinds = []string{"ar", "al", "pl", "ra"} + +// Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a +// change means previously resolved (or absent) state may no longer be correct. +func Fingerprint() string { + raw := fmt.Sprintf("%s|%s|%s|%t|%s", + conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, + conf.Server.Agents, conf.Server.EnableExternalServices, consts.Version) + sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive + return hex.EncodeToString(sum[:]) +} + +// Backfill enqueues artwork resolution for every entity when the config fingerprint changed +// (or was never stored), artists first so those pages resolve before the larger backlog. +func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { + current := Fingerprint() + props := ds.Property(ctx) + stored, err := props.DefaultGet(FingerprintPropertyKey, "") + if err != nil { + return false, err + } + if stored == current { + return false, nil + } + + artists, err := ds.Artist(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "ar", idsOf(artists, func(a model.Artist) string { return a.ID })); err != nil { + return false, err + } + + albums, err := ds.Album(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "al", idsOf(albums, func(a model.Album) string { return a.ID })); err != nil { + return false, err + } + + playlists, err := ds.Playlist(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "pl", idsOf(playlists, func(p model.Playlist) string { return p.ID })); err != nil { + return false, err + } + + radios, err := ds.Radio(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "ra", idsOf(radios, func(r model.Radio) string { return r.ID })); err != nil { + return false, err + } + + if err := props.Put(FingerprintPropertyKey, current); err != nil { + return false, err + } + log.Info(ctx, "Artwork: config fingerprint changed, backfill enqueued") + return true, nil +} + +func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, ids []string) error { + if len(ids) == 0 { + return nil + } + items := make([]model.ArtworkQueueItem, len(ids)) + for i, id := range ids { + items[i] = model.ArtworkQueueItem{ + ItemKind: kind, ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill, + } + } + return ds.ArtworkQueue(ctx).Enqueue(items...) +} + +func idsOf[T any](items []T, id func(T) string) []string { + ids := make([]string, len(items)) + for i, it := range items { + ids[i] = id(it) + } + return ids +} + +// EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across +// every artwork-bearing kind, for the periodic recheck job. +func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error { + cutoff := time.Now().Add(-staleAbsentAge) + queue := ds.ArtworkQueue(ctx) + for _, kind := range staleAbsentKinds { + if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil { + return err + } + } + return nil +} diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go new file mode 100644 index 000000000..8272514b0 --- /dev/null +++ b/core/artwork/housekeeping_test.go @@ -0,0 +1,174 @@ +package artwork + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can +// assert phase ordering (artists-first) that same-priority timestamps can't guarantee. +type orderTrackingQueueRepo struct { + *tests.MockArtworkQueueRepo + callKinds []string +} + +func (o *orderTrackingQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { + if len(items) > 0 { + o.callKinds = append(o.callKinds, items[0].ItemKind) + } + return o.MockArtworkQueueRepo.Enqueue(items...) +} + +var _ = Describe("Housekeeping", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + queueRepo *orderTrackingQueueRepo + propRepo *tests.MockedPropertyRepo + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + conf.Server.CoverArtPriority = "embedded, folder" + conf.Server.ArtistArtPriority = "artist.jpg" + conf.Server.Agents = "spotify" + conf.Server.EnableExternalServices = true + + queueRepo = &orderTrackingQueueRepo{MockArtworkQueueRepo: tests.CreateMockArtworkQueueRepo()} + propRepo = &tests.MockedPropertyRepo{} + ds = &tests.MockDataStore{MockedArtworkQueue: queueRepo, MockedProperty: propRepo} + }) + + seedEntities := func() { + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar1"}, {ID: "ar2"}}) + ds.MockedArtist = artistRepo + + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{{ID: "al1"}}) + ds.MockedAlbum = albumRepo + + playlistRepo := tests.CreateMockPlaylistRepo() + playlistRepo.SetData(model.Playlists{{ID: "pl1"}}) + ds.MockedPlaylist = playlistRepo + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.All = model.Radios{{ID: "ra1"}} + ds.MockedRadio = radioRepo + } + + Describe("Fingerprint", func() { + It("changes when a fingerprint-affecting config value changes", func() { + f1 := Fingerprint() + conf.Server.CoverArtPriority = "folder, embedded" + f2 := Fingerprint() + Expect(f1).NotTo(Equal(f2)) + }) + }) + + Describe("Backfill", func() { + It("enqueues nothing and returns false when the stored fingerprint matches", func() { + seedEntities() + Expect(propRepo.Put(FingerprintPropertyKey, Fingerprint())).To(Succeed()) + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeFalse()) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + }) + + It("runs the backfill when no fingerprint was ever stored", func() { + seedEntities() + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(int64(5))) // 2 artists + 1 album + 1 playlist + 1 radio + + stored, err := propRepo.Get(FingerprintPropertyKey) + Expect(err).ToNot(HaveOccurred()) + Expect(stored).To(Equal(Fingerprint())) + }) + + It("enqueues artists before albums/playlists/radios, all at Backfill priority", func() { + seedEntities() + Expect(propRepo.Put(FingerprintPropertyKey, "stale-fingerprint")).To(Succeed()) + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + + Expect(queueRepo.callKinds).ToNot(BeEmpty()) + artistCallIdx := -1 + for i, k := range queueRepo.callKinds { + if k == "ar" { + artistCallIdx = i + break + } + } + Expect(artistCallIdx).To(Equal(0), "artists must be the first Enqueue call") + for i, k := range queueRepo.callKinds { + if k != "ar" { + Expect(i).To(BeNumerically(">", artistCallIdx)) + } + } + + for _, it := range queueRepo.Data { + Expect(it.Priority).To(Equal(model.ArtworkPriorityBackfill)) + Expect(it.ItemKind).To(BeElementOf("ar", "al", "pl", "ra")) + } + }) + }) + + Describe("EnqueueStaleAbsentAll", func() { + var artRepo *tests.MockArtworkRepo + + BeforeEach(func() { + artRepo = tests.CreateMockArtworkRepo() + ds.MockedArtwork = artRepo + queueRepo.ItemArtworkSource = artRepo + }) + + It("enqueues only absent entries older than the recheck window, across all kinds", func() { + old := time.Now().Add(-48 * time.Hour) + recent := time.Now().Add(-time.Hour) + + artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["pl-stale"] = model.ItemArtwork{ItemKind: "pl", ItemID: "pl1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["ra-stale"] = model.ItemArtwork{ItemKind: "ra", ItemID: "ra1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + // Not stale: too recent. + artRepo.ItemData["ar-recent"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar2", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: recent} + // Not absent: has a resolved hash. + artRepo.ItemData["al-resolved"] = model.ItemArtwork{ItemKind: "al", ItemID: "al2", ImageType: model.ImageTypePrimary, Hash: "somehash", AttemptedAt: old} + + err := EnqueueStaleAbsentAll(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + + Expect(queueRepo.Data).To(HaveLen(4)) + for _, it := range queueRepo.Data { + Expect(it.Priority).To(Equal(model.ArtworkPriorityRecheck)) + } + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "pl1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ra", "ra1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil()) + }) + }) +}) From 1ed8ebf9b0ea47643cd87f637fec38057ab3bfd3 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 11:16:46 -0400 Subject: [PATCH 30/51] test(artwork): leak/soak coverage and deferred assertions --- core/artwork/artwork_suite_test.go | 17 +-- core/artwork/image_store_test.go | 13 ++- core/artwork/prune_test.go | 49 +++++++++ core/artwork/worker_soak_test.go | 159 +++++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 core/artwork/worker_soak_test.go diff --git a/core/artwork/artwork_suite_test.go b/core/artwork/artwork_suite_test.go index 1fd04627a..b721200e1 100644 --- a/core/artwork/artwork_suite_test.go +++ b/core/artwork/artwork_suite_test.go @@ -19,12 +19,17 @@ import ( ) func TestArtwork(t *testing.T) { - // Only run goleak checks when the GOLEAK env var is set - if os.Getenv("GOLEAK") != "" { - defer goleak.VerifyNone(t, - goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"), - ) - } + // Runs unconditionally: the two leaks below are pre-existing and out of this + // package's control, so they're ignored by exact top-function instead. + defer goleak.VerifyNone(t, + goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"), + // notify's own init() starts this dispatcher the moment it's imported + // (via core/storage/local or plugins); it never exits. + goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*recursiveTree).dispatch"), + // The old cache_warmer.go starts a goroutine per NewCacheWarmer call with + // no shutdown path (dark-launch target for Phase 2, not touched here). + goleak.IgnoreTopFunction("github.com/navidrome/navidrome/core/artwork.(*cacheWarmer).waitSignal"), + ) tests.Init(t, false) log.SetLevel(log.LevelFatal) diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go index 214cab3ad..6537ca906 100644 --- a/core/artwork/image_store_test.go +++ b/core/artwork/image_store_test.go @@ -44,11 +44,20 @@ var _ = Describe("ImageStore", func() { Expect(got).To(Equal(data)) }) - It("is idempotent on duplicate writes", func() { + It("is idempotent on duplicate writes and preserves the original content", func() { data := []byte("dup") h, _ := HashImage(bytes.NewReader(data)) Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) - Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + // A duplicate write only touches mtime; passing different bytes under the same + // hash proves the second reader is never consumed to overwrite the file. + Expect(store.Write(h, "image/png", bytes.NewReader([]byte("not-dup")))).To(Succeed()) + + rc, err := store.Open(h, "image/png") + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + got, err := io.ReadAll(rc) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(data)) }) It("refreshes the mtime on a duplicate write", func() { diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 983e7c555..3de46942a 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "os" + "path/filepath" "time" "github.com/navidrome/navidrome/model" @@ -171,6 +172,54 @@ var _ = Describe("Prune", func() { rc.Close() }) + It("warns and continues past a store.Remove failure instead of aborting the loop", func() { + tests.SkipOnWindows("uses Unix file permission bits") + old := time.Now().Add(-2 * time.Hour) + + blocked := []byte("blocked-bytes") + hb, _ := HashImage(bytes.NewReader(blocked)) + Expect(store.Write(hb, "image/jpeg", bytes.NewReader(blocked))).To(Succeed()) + Expect(os.Chtimes(store.path(hb, "image/jpeg"), old, old)).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hb, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(hb, old) + + good := []byte("good-bytes") + hg, _ := HashImage(bytes.NewReader(good)) + Expect(store.Write(hg, "image/jpeg", bytes.NewReader(good))).To(Succeed()) + Expect(os.Chtimes(store.path(hg, "image/jpeg"), old, old)).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hg, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(hg, old) + + // A read-only shard directory makes os.Remove fail (EACCES) for hb's file only. + shardDir := filepath.Dir(store.path(hb, "image/jpeg")) + Expect(os.Chmod(shardDir, 0500)).To(Succeed()) + DeferCleanup(func() { _ = os.Chmod(shardDir, 0755) }) + + // hb (blocked) is processed first: if store.Remove's failure aborted the loop + // instead of warning and continuing, hg would never be reached. + awRepo.OrphanHashes = []string{hb, hg} + + // Prune still errors: Sweep independently revisits hb's leftover file and, + // unlike the loop below, has no warn-and-continue fallback of its own. + err := Prune(context.Background(), ds, store) + Expect(err).To(HaveOccurred()) + + // hg: reached and fully pruned despite being queued after the failing hb - + // proof the loop didn't return/break on the first Remove error. + _, err = awRepo.GetImage(hg) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = store.Open(hg, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + + // hb: row still purged (DeleteOrphans doesn't depend on file removal), but the + // file itself survives since store.Remove failed and only warned. + _, err = awRepo.GetImage(hb) + Expect(err).To(MatchError(model.ErrNotFound)) + rc, err := store.Open(hb, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + It("never sweeps files on a transient DB error", func() { ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()} diff --git a/core/artwork/worker_soak_test.go b/core/artwork/worker_soak_test.go new file mode 100644 index 000000000..f6ecef422 --- /dev/null +++ b/core/artwork/worker_soak_test.go @@ -0,0 +1,159 @@ +package artwork + +import ( + "context" + "io" + "os" + "runtime" + "testing" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" +) + +// soakCycles is deliberately >2000: this is a leak regression guard, not a +// performance benchmark, so it favors a stable signal over raw speed. +const soakCycles = 2200 + +// TestWorkerSoak drives processItem across a mix of sources (folder, embedded +// extraction, dangling refs) for many cycles, reading each acquired image back +// through ImageStore.Open, and asserts goroutines/heap plateau instead of +// growing unbounded. Skipped under -short. +func TestWorkerSoak(t *testing.T) { + if testing.Short() { + t.Skip("skipping soak test in short mode") + } + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + defer configtest.SetupConfig()() + + repoRoot, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + folderRepo := &fakeFolderRepo{result: []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }}} + ffm := tests.NewMockFFmpeg("") + prov := &fakeExternalProvider{} + artRepo := tests.CreateMockArtworkRepo() + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{ + {ID: "al-folder", Name: "Folder Album", FolderIDs: []string{"f1"}}, + {ID: "al-embed", Name: "Embedded Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + ds := &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + MockedAlbum: albumRepo, + } + store := NewImageStore(t.TempDir()) + deps := &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm} + conf.Server.CoverArtPriority = "cover.jpg, embedded" + + // Dangling refs (al/ra ids the repos don't know about) mirror an entity + // deleted after being enqueued; ds.Radio auto-provisions an empty mock repo. + items := []model.ArtworkQueueItem{ + {ItemKind: "al", ItemID: "al-folder"}, + {ItemKind: "al", ItemID: "al-embed"}, + {ItemKind: "al", ItemID: "al-does-not-exist"}, + {ItemKind: "ra", ItemID: "ra-does-not-exist"}, + } + + fdCount := func() int { + if runtime.GOOS != "linux" { + return -1 + } + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + return -1 + } + return len(entries) + } + + settleGoroutines := func() int { + // Background goroutines (GC workers, etc.) can take a moment to wind down; + // poll for two consecutive equal samples instead of trusting a single one. + prev := -1 + for range 100 { + runtime.GC() + n := runtime.NumGoroutine() + if n == prev { + return n + } + prev = n + time.Sleep(10 * time.Millisecond) + } + return prev + } + + baselineGoroutines := settleGoroutines() + baselineFDs := fdCount() + + var heapAt10Pct uint64 + start := time.Now() + for i := range soakCycles { + it := items[i%len(items)] + out := processItem(context.Background(), deps, it) + + // "Serve-adjacent" read-back: exercise the Phase 2 surfaces a caller would + // use after acquisition, not the old serving pipeline. + if out == outcomeFound { + ia, err := artRepo.GetItemArtwork(it.ItemKind, it.ItemID, model.ImageTypePrimary) + if err != nil { + t.Fatalf("cycle %d: GetItemArtwork: %v", i, err) + } + art, err := artRepo.GetImage(ia.Hash) + if err != nil { + t.Fatalf("cycle %d: GetImage: %v", i, err) + } + rc, err := store.Open(ia.Hash, art.Mime) + switch { + case err == nil: + _, _ = io.Copy(io.Discard, rc) + rc.Close() + case os.IsNotExist(err): + // Folder-backed art has no store file; that's expected. + default: + t.Fatalf("cycle %d: store.Open: %v", i, err) + } + } + + if i == soakCycles/10 { + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + heapAt10Pct = ms.HeapAlloc + } + } + elapsed := time.Since(start) + + finalGoroutines := settleGoroutines() + finalFDs := fdCount() + + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + + t.Logf("soak: cycles=%d elapsed=%s goroutines(baseline=%d final=%d) heap(10%%-mark=%d final=%d) fds(baseline=%d final=%d)", + soakCycles, elapsed, baselineGoroutines, finalGoroutines, heapAt10Pct, ms.HeapAlloc, baselineFDs, finalFDs) + + if finalGoroutines > baselineGoroutines { + t.Errorf("goroutine count grew: baseline=%d final=%d", baselineGoroutines, finalGoroutines) + } + if heapAt10Pct > 0 && ms.HeapAlloc > 2*heapAt10Pct { + t.Errorf("heap did not plateau: 10%%-mark=%d final=%d (final > 2x 10%%-mark)", heapAt10Pct, ms.HeapAlloc) + } + if runtime.GOOS == "linux" && baselineFDs >= 0 && finalFDs > baselineFDs { + t.Errorf("fd count grew: baseline=%d final=%d", baselineFDs, finalFDs) + } +} From c01e9b31846618517c65050bf0ef4c0f9c461236 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 11:57:25 -0400 Subject: [PATCH 31/51] fix(artwork): propagate transient artist image errors to the worker callGetImage swallowed all agent errors, so an agent outage surfaced as ErrNotFound and the worker settled artist artwork as a definitive absent (and reset the breaker). Add an additive ArtistImageResult path that returns the underlying agent error on transient failure while keeping ArtistImage byte-identical for existing callers; the worker's artist external step uses it via fromArtistExternalResult. --- core/artwork/e2e/helpers_test.go | 3 ++ core/artwork/sources.go | 12 ++++++ core/external/provider.go | 42 +++++++++++++++++---- core/external/provider_artistimage_test.go | 43 ++++++++++++++++++++++ server/subsonic/e2e/e2e_suite_test.go | 4 ++ 5 files changed, 96 insertions(+), 8 deletions(-) diff --git a/core/artwork/e2e/helpers_test.go b/core/artwork/e2e/helpers_test.go index e3abca097..4e99f7a5c 100644 --- a/core/artwork/e2e/helpers_test.go +++ b/core/artwork/e2e/helpers_test.go @@ -177,6 +177,9 @@ func (n *noopProvider) TopSongs(context.Context, string, int) (model.MediaFiles, func (n *noopProvider) ArtistImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } +func (n *noopProvider) ArtistImageResult(context.Context, string) (*url.URL, error) { + return nil, model.ErrNotFound +} func (n *noopProvider) AlbumImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } diff --git a/core/artwork/sources.go b/core/artwork/sources.go index 04a9257fb..d4ad0a2a7 100644 --- a/core/artwork/sources.go +++ b/core/artwork/sources.go @@ -198,6 +198,18 @@ func fromArtistExternalSource(ctx context.Context, ar model.Artist, provider ext } } +// fromArtistExternalResult is the worker's artist external step: via ArtistImageResult a +// transient agent failure surfaces as an error (extError) rather than settling as absent. +func fromArtistExternalResult(ctx context.Context, ar model.Artist, provider external.Provider) sourceFunc { + return func() (io.ReadCloser, string, error) { + imageUrl, err := provider.ArtistImageResult(ctx, ar.ID) + if err != nil { + return nil, "", err + } + return fromURL(ctx, imageUrl) + } +} + func fromAlbumExternalSource(ctx context.Context, al model.Album, provider external.Provider) sourceFunc { return func() (io.ReadCloser, string, error) { imageUrl, err := provider.AlbumImage(ctx, al.ID) diff --git a/core/external/provider.go b/core/external/provider.go index 459e8a205..42da7eb45 100644 --- a/core/external/provider.go +++ b/core/external/provider.go @@ -36,6 +36,8 @@ type Provider interface { SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error) TopSongs(ctx context.Context, artist string, count int) (model.MediaFiles, error) ArtistImage(ctx context.Context, id string) (*url.URL, error) + // ArtistImageResult is like ArtistImage but reports a transient agent failure as a real error, not ErrNotFound. + ArtistImageResult(ctx context.Context, id string) (*url.URL, error) AlbumImage(ctx context.Context, id string) (*url.URL, error) } @@ -258,7 +260,7 @@ func (e *provider) populateArtistInfo(ctx context.Context, artist auxArtist) (au // Call all registered agents and collect information g := errgroup.Group{} g.SetLimit(2) - g.Go(func() error { e.callGetImage(ctx, e.ag, &artist); return nil }) + g.Go(func() error { _ = e.callGetImage(ctx, e.ag, &artist); return nil }) g.Go(func() error { e.callGetBiography(ctx, e.ag, &artist); return nil }) g.Go(func() error { e.callGetURL(ctx, e.ag, &artist); return nil }) g.Go(func() error { e.callGetSimilarArtists(ctx, e.ag, &artist, maxSimilarArtists, true); return nil }) @@ -371,18 +373,35 @@ func (e *provider) similarSongsFallback(ctx context.Context, id string, count in } func (e *provider) ArtistImage(ctx context.Context, id string) (*url.URL, error) { + u, _, err := e.artistImage(ctx, id) + return u, err +} + +// ArtistImageResult is like ArtistImage but surfaces a transient agent failure as the +// real error, so an agent outage is not mistaken for a definitive no-image (ErrNotFound). +func (e *provider) ArtistImageResult(ctx context.Context, id string) (*url.URL, error) { + u, agentErr, err := e.artistImage(ctx, id) + if agentErr != nil && errors.Is(err, model.ErrNotFound) { + return nil, agentErr + } + return u, err +} + +// artistImage returns the agent error (agentErr) separately from the caller-facing err, +// so ArtistImageResult can tell "agent errored" apart from "definitively no image". +func (e *provider) artistImage(ctx context.Context, id string) (u *url.URL, agentErr error, err error) { artist, err := e.getArtist(ctx, id) if err != nil { - return nil, err + return nil, nil, err } imageUrl := artist.ArtistImageUrl() if imageUrl == "" { // No cached URL — must fetch from external source synchronously - e.callGetImage(ctx, e.ag, &artist) + agentErr = e.callGetImage(ctx, e.ag, &artist) if utils.IsCtxDone(ctx) { log.Warn(ctx, "ArtistImage call canceled", ctx.Err()) - return nil, ctx.Err() + return nil, agentErr, ctx.Err() } imageUrl = artist.ArtistImageUrl() } else { @@ -396,9 +415,10 @@ func (e *provider) ArtistImage(ctx context.Context, id string) (*url.URL, error) } if imageUrl == "" { - return nil, model.ErrNotFound + return nil, agentErr, model.ErrNotFound } - return url.Parse(imageUrl) + u, err = url.Parse(imageUrl) + return u, agentErr, err } func (e *provider) AlbumImage(ctx context.Context, id string) (*url.URL, error) { @@ -519,10 +539,15 @@ func (e *provider) callGetBiography(ctx context.Context, agent agents.ArtistBiog artist.Biography = strings.ReplaceAll(bio, " images[j].Size }) @@ -535,6 +560,7 @@ func (e *provider) callGetImage(ctx context.Context, agent agents.ArtistImageRet if len(images) >= 3 { artist.SmallImageUrl = images[2].URL } + return nil } func (e *provider) callGetSimilarArtists(ctx context.Context, agent agents.ArtistSimilarRetriever, artist *auxArtist, diff --git a/core/external/provider_artistimage_test.go b/core/external/provider_artistimage_test.go index 79612d651..621aac574 100644 --- a/core/external/provider_artistimage_test.go +++ b/core/external/provider_artistimage_test.go @@ -330,6 +330,49 @@ var _ = Describe("Provider - ArtistImage", func() { Expect(logBuf.String()).To(ContainSubstring("Artist image info expired, enqueuing background refresh")) }) + Describe("ArtistImageResult", func() { + It("returns the real agent error on a transient failure, not ErrNotFound", func() { + agentErr := errors.New("agent timed out") + mockImageAgent.Mock = mock.Mock{} + mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return(nil, agentErr).Once() + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).To(MatchError(agentErr)) + Expect(err).ToNot(MatchError(model.ErrNotFound)) + Expect(imgURL).To(BeNil()) + }) + + It("returns ErrNotFound when the agent definitively has no image", func() { + mockImageAgent.Mock = mock.Mock{} + mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return(nil, agents.ErrNotFound).Once() + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).To(MatchError(model.ErrNotFound)) + Expect(imgURL).To(BeNil()) + }) + + It("returns ErrNotFound when the agent returns no images without error", func() { + mockImageAgent.Mock = mock.Mock{} + mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return([]agents.ExternalImage{}, nil).Once() + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).To(MatchError(model.ErrNotFound)) + Expect(imgURL).To(BeNil()) + }) + + It("returns the largest image URL on success", func() { + expectedURL, _ := url.Parse("http://example.com/large.jpg") + + imgURL, err := provider.ArtistImageResult(ctx, "artist-1") + + Expect(err).ToNot(HaveOccurred()) + Expect(imgURL).To(Equal(expectedURL)) + }) + }) + Context("Unicode handling in artist names", func() { var artistWithEnDash *model.Artist var expectedURL *url.URL diff --git a/server/subsonic/e2e/e2e_suite_test.go b/server/subsonic/e2e/e2e_suite_test.go index 6875b6370..98ad6d1ee 100644 --- a/server/subsonic/e2e/e2e_suite_test.go +++ b/server/subsonic/e2e/e2e_suite_test.go @@ -361,6 +361,10 @@ func (n noopProvider) ArtistImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } +func (n noopProvider) ArtistImageResult(context.Context, string) (*url.URL, error) { + return nil, model.ErrNotFound +} + func (n noopProvider) AlbumImage(context.Context, string) (*url.URL, error) { return nil, model.ErrNotFound } From 454fd248339484ad415a79a64284b3b5eb544555 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 11:57:32 -0400 Subject: [PATCH 32/51] fix(artwork): resolve full playlist source chain resolvePlaylist only built the generated grid, dropping the uploaded-image, sidecar and ExternalImageURL sources the old reader_playlist.go chain serves. Port the full chain before the grid fallback: uploaded (upload), sidecar (folder), and ExternalImageURL routed through extGate with the same extError semantics as the other external steps. Also rewires the artist external step onto ArtistImageResult. --- core/artwork/resolve.go | 46 +++++++++++++++++++++++--- core/artwork/resolve_test.go | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 5aafdf5c5..272234c9d 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -10,6 +10,7 @@ import ( "image/png" "io" "io/fs" + "net/url" "os" "strings" @@ -29,7 +30,7 @@ type resolution struct { extError bool // an external source errored/timed out (forces failed, never absent) } -// extGateFunc is an alias for the external-step wrapper Task 4 injects (rate +// extGateFunc is an alias for the external-step wrapper the worker injects (rate // limiter + circuit breaker); resolveItem defaults to a plain passthrough. type extGateFunc = func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) @@ -137,7 +138,7 @@ func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provid pattern = strings.TrimSpace(pattern) switch { case pattern == "external": - if res, ok, isErr := resolveExternalStep(extGate, fromArtistExternalSource(ctx, *ar, prov)); ok { + if res, ok, isErr := resolveExternalStep(extGate, fromArtistExternalResult(ctx, *ar, prov)); ok { return res, nil } else if isErr { extErr = true @@ -165,20 +166,33 @@ func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provid return resolution{extError: extErr}, nil } -// resolvePlaylist ports the 2x2 generated grid from reader_playlist.go, -// sourcing tiles through resolveAlbum instead of the old cached reader. +// resolvePlaylist ports reader_playlist.go's chain: uploaded image, sidecar, +// ExternalImageURL, then the generated 2x2 grid sourced through resolveAlbum. func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, playlistID string, extGate extGateFunc) (resolution, error) { pl, err := ds.Playlist(ctx).Get(playlistID) if err != nil { return resolution{}, err } + + var extErr bool + if res, ok := resolveLocalFile(pl.UploadedImagePath(), "upload"); ok { + return res, nil + } + if res, ok := resolveLocalFile(findPlaylistSidecarPath(ctx, pl.Path), "folder"); ok { + return res, nil + } + if res, ok, isErr := resolveExternalStep(extGate, fromPlaylistExternalSource(ctx, *pl)); ok { + return res, nil + } else if isErr { + extErr = true + } + albumIDs, err := ds.Playlist(ctx).Tracks(pl.ID, false).GetAlbumIDs(model.QueryOptions{Max: 4, Sort: "random()"}) if err != nil { return resolution{}, err } var tiles []image.Image - var extErr bool var tileErr error // first internal (non-external) tile failure, e.g. album deleted mid-flight for _, albumID := range albumIDs { res, err := resolveAlbum(ctx, ds, prov, ffm, albumID, extGate) @@ -246,6 +260,28 @@ func resolveExternalStep(extGate extGateFunc, sf func() (io.ReadCloser, string, return resolution{}, false, err != nil && !errors.Is(err, model.ErrNotFound) } +// fromPlaylistExternalSource mirrors reader_playlist.go's ExternalImageURL step: +// a remote URL (gated) when M3U external art is enabled, else a local file path. +func fromPlaylistExternalSource(ctx context.Context, pl model.Playlist) sourceFunc { + return func() (io.ReadCloser, string, error) { + imgURL := pl.ExternalImageURL + if imgURL == "" { + return nil, "", nil + } + parsed, err := url.Parse(imgURL) + if err != nil { + return nil, "", err + } + if parsed.Scheme == "http" || parsed.Scheme == "https" { + if !conf.Server.EnableM3UExternalAlbumArt { + return nil, "", nil + } + return fromURL(ctx, parsed) + } + return fromLocalFile(imgURL)() + } +} + func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, embedRel string) (resolution, bool) { if embedRel == "" { return resolution{}, false diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index c4ccc3733..cab757dbb 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -40,6 +40,10 @@ func (f *fakeExternalProvider) ArtistImage(ctx context.Context, id string) (*url return nil, model.ErrNotFound } +func (f *fakeExternalProvider) ArtistImageResult(ctx context.Context, id string) (*url.URL, error) { + return f.ArtistImage(ctx, id) +} + var _ = Describe("resolveItem", func() { var ( ctx context.Context @@ -334,6 +338,66 @@ var _ = Describe("resolveItem", func() { Entry("4 albums -> full grid", []string{"t1", "t2", "t3", "t4"}, tileSize-1), ) + It("resolves the uploaded image before the generated grid", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "playlist"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "playlist", "plu_test.jpg") + Expect(os.WriteFile(imgPath, []byte("uploaded playlist image"), 0600)).To(Succeed()) + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plu", Name: "Playlist", UploadedImage: "plu_test.jpg"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plu"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("upload")) + Expect(res.sourcePath).To(Equal(imgPath)) + }) + + It("resolves a sidecar image next to the playlist file before the grid", func() { + plDir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(plDir, "list.m3u"), []byte("#EXTM3U"), 0600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(plDir, "list.jpg"), []byte("sidecar image"), 0600)).To(Succeed()) + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pls", Name: "Playlist", Path: filepath.Join(plDir, "list.m3u")}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pls"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("list.jpg")) + }) + + It("routes ExternalImageURL through extGate and sets extError on transient failure", func() { + conf.Server.EnableM3UExternalAlbumArt = true + folderRepo.result = nil // no grid tiles, so the external failure is what surfaces + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "ple", Name: "Playlist", ExternalImageURL: "http://example.com/cover.jpg"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + var extGateCalls int + extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + extGateCalls++ + return nil, "", errors.New("network down") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "ple"}, extGate) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + Expect(extGateCalls).To(Equal(1)) + }) + It("yields an empty resolution when no album has art", func() { ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ {ID: "empty1", Name: "Empty"}, From bab9b5cd3a6c508d4f7318ac07bf6fde0ff64979 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 11:57:40 -0400 Subject: [PATCH 33/51] fix(artwork): purge dangling queue rows and guard concurrent re-enqueues Queue rows for deleted entities failed forever (Get -> ErrNotFound -> failed -> capped retries, unbounded). Add ArtworkQueueRepository.PurgeDangling, called from Prune next to the item_artwork purge. Separately, the found/absent path unconditionally deleted the dequeued row, erasing a concurrent scan re-enqueue; switch to DeleteIfUnchanged, which deletes only while retry_at still matches the dequeued value (verified retry_at is the column an Enqueue upsert resets). --- core/artwork/prune.go | 9 ++++ core/artwork/prune_test.go | 15 ++++++ core/artwork/worker.go | 13 +++--- core/artwork/worker_test.go | 47 +++++++++++++++++++ model/artwork.go | 5 ++ persistence/artwork_queue_repository.go | 23 +++++++++ persistence/artwork_queue_repository_test.go | 49 ++++++++++++++++++++ tests/mock_artwork_queue_repo.go | 31 +++++++++++++ 8 files changed, 186 insertions(+), 6 deletions(-) diff --git a/core/artwork/prune.go b/core/artwork/prune.go index 5278beaa4..35eddb525 100644 --- a/core/artwork/prune.go +++ b/core/artwork/prune.go @@ -22,6 +22,15 @@ func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { log.Info(ctx, "Prune: purged dangling item artwork state", "count", purged) } + // Queue rows for deleted entities would otherwise retry forever (Get -> not found -> failed). + queuePurged, err := ds.ArtworkQueue(ctx).PurgeDangling() + if err != nil { + return err + } + if queuePurged > 0 { + log.Info(ctx, "Prune: purged dangling artwork queue rows", "count", queuePurged) + } + // One grace cutoff for both the DB orphan check and the file sweep: files younger // than the window may belong to acquisitions whose rows aren't committed yet. cutoff := time.Now().Add(-pruneMinAge) diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 3de46942a..5511b2305 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -59,6 +59,21 @@ var _ = Describe("Prune", func() { Expect(err).ToNot(HaveOccurred()) }) + It("purges dangling artwork_queue rows for gone entities", func() { + queueRepo := tests.CreateMockArtworkQueueRepo() + Expect(queueRepo.Enqueue( + model.ArtworkQueueItem{ItemKind: "al", ItemID: "gone-album", ImageType: model.ImageTypePrimary}, + model.ArtworkQueueItem{ItemKind: "al", ItemID: "live-album", ImageType: model.ImageTypePrimary}, + )).To(Succeed()) + queueRepo.ExistingIDs = map[string]map[string]bool{"al": {"live-album": true}} + ds.MockedArtworkQueue = queueRepo + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + Expect(findQueued(queueRepo, "al", "gone-album")).To(BeNil()) + Expect(findQueued(queueRepo, "al", "live-album")).ToNot(BeNil()) + }) + It("deletes orphan rows and their store files, keeps referenced ones", func() { data := []byte("orphan-bytes") h, _ := HashImage(bytes.NewReader(data)) diff --git a/core/artwork/worker.go b/core/artwork/worker.go index 1ebe15527..fb20a2bbc 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -27,9 +27,8 @@ const ( var errBreakerOpen = errors.New("artwork: external circuit breaker open") -// Worker drains the artwork queue and runs each item through processItem. The -// external step is rate-limited and circuit-broken; prune is serialized against -// in-flight acquisitions via pruneMu (acquisitions RLock, prune Lock). +// Worker drains the artwork queue through processItem: the external step is rate-limited +// and circuit-broken, and prune is serialized against in-flight acquisitions via pruneMu. type Worker struct { deps workerDeps limiter *rate.Limiter @@ -150,7 +149,9 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) { queue := w.deps.ds.ArtworkQueue(ctx) switch out { case outcomeFound, outcomeAbsent: - if err := queue.Delete(item.ItemKind, item.ItemID, item.ImageType); err != nil { + // DeleteIfUnchanged, not Delete: a scan that re-enqueued this row mid-flight reset + // its retry_at, so the row survives here and the next drain re-resolves it. + if err := queue.DeleteIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt); err != nil { log.Warn(ctx, "artwork: could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err) } case outcomeFailed: @@ -161,8 +162,8 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) { } } -// claim reserves items not already in flight, so a wake-triggered re-drain never -// double-processes an item still running from a previous cycle. +// claim reserves items not already in flight, so a row appearing twice within a single +// batch is processed once. func (w *Worker) claim(batch []model.ArtworkQueueItem) []model.ArtworkQueueItem { w.mu.Lock() defer w.mu.Unlock() diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index d41249fa2..058d707b8 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -16,6 +16,27 @@ import ( . "github.com/onsi/gomega" ) +// reenqueueOnDequeue simulates a concurrent scan Enqueue between DequeueBatch and the +// worker's delete by bumping retry_at, so a DeleteIfUnchanged on the dequeued value no-ops. +type reenqueueOnDequeue struct { + *tests.MockArtworkQueueRepo + done bool +} + +func (r *reenqueueOnDequeue) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + items, err := r.MockArtworkQueueRepo.DequeueBatch(n) + if !r.done && len(items) > 0 { + r.done = true + for k, it := range r.Data { + if it.ItemKind == items[0].ItemKind && it.ItemID == items[0].ItemID { + it.RetryAt = items[0].RetryAt.Add(time.Minute) + r.Data[k] = it + } + } + } + return items, err +} + func findQueued(q *tests.MockArtworkQueueRepo, kind, id string) *model.ArtworkQueueItem { for _, it := range q.Data { if it.ItemKind == kind && it.ItemID == id { @@ -114,6 +135,32 @@ var _ = Describe("Worker", func() { Expect(err).To(MatchError(model.ErrNotFound), "a timeout must never settle on absent") }) + It("keeps a row re-enqueued between dequeue and delete", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al7", Name: "Album", FolderIDs: []string{"f1"}}, + }) + racing := &reenqueueOnDequeue{MockArtworkQueueRepo: queueRepo} + ds.MockedArtworkQueue = racing + w = NewWorker(ds, store, prov, ffm) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ + ItemKind: "al", ItemID: "al7", Priority: model.ArtworkPriorityScan, + })).To(Succeed()) + + n, err := w.drain(ctx, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + // The concurrent re-enqueue changed retry_at, so the found-path delete was a no-op. + Expect(findQueued(queueRepo, "al", "al7")).ToNot(BeNil()) + ia, err := artRepo.GetItemArtwork("al", "al7", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder")) + }) + It("returns zero when the queue is empty", func() { n, err := w.drain(ctx, 2) Expect(err).ToNot(HaveOccurred()) diff --git a/model/artwork.go b/model/artwork.go index 73db00ca5..0aec6489a 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -89,7 +89,12 @@ type ArtworkQueueRepository interface { // MarkFailed increments attempts and pushes retry_at into the future. MarkFailed(kind, id, imageType string, retryAt time.Time) error Delete(kind, id, imageType string) error + // DeleteIfUnchanged deletes the row only if its retry_at still matches retryAt, so a + // concurrent re-enqueue (which resets retry_at) survives instead of being erased. + DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error Count() (int64, error) // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) + // PurgeDangling removes queue rows whose entity no longer exists. + PurgeDangling() (int64, error) } diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go index ced5e9913..eb5f21ecd 100644 --- a/persistence/artwork_queue_repository.go +++ b/persistence/artwork_queue_repository.go @@ -70,6 +70,29 @@ func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) } +// DeleteIfUnchanged deletes the row only while its retry_at still equals the dequeued +// value; a concurrent Enqueue resets retry_at, so the row survives to be re-resolved. +func (r *artworkQueueRepository) DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error { + return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": retryAt}) +} + +// PurgeDangling removes queue rows whose entity no longer exists, per kind. +func (r *artworkQueueRepository) PurgeDangling() (int64, error) { + var total int64 + for kind, table := range danglingItemArtworkKinds { + del := Delete(r.tableName).Where(And{ + Eq{"item_kind": kind}, + Expr("item_id NOT IN (SELECT id FROM " + table + ")"), + }) + c, err := r.executeSQL(del) + if err != nil { + return total, err + } + total += c + } + return total, nil +} + func (r *artworkQueueRepository) Count() (int64, error) { var res struct{ Count int64 } err := r.queryOne(Select("count(*) as count").From(r.tableName), &res) diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go index 2b6f0d8c5..e96802284 100644 --- a/persistence/artwork_queue_repository_test.go +++ b/persistence/artwork_queue_repository_test.go @@ -63,6 +63,55 @@ var _ = Describe("ArtworkQueueRepository", func() { Expect(n).To(BeZero()) }) + It("DeleteIfUnchanged deletes only while retry_at is unchanged", func() { + Expect(repo.Enqueue(item("al", "d1", model.ArtworkPriorityScan))).To(Succeed()) + // Anchor retry_at in the past so it can never collide with the re-enqueue's now. + Expect(repo.MarkFailed("al", "d1", model.ImageTypePrimary, time.Now().Add(-time.Hour))).To(Succeed()) + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(1)) + original := got[0].RetryAt + + // A concurrent scan re-enqueues, resetting retry_at to now. + Expect(repo.Enqueue(item("al", "d1", model.ArtworkPriorityScan))).To(Succeed()) + + // Deleting with the stale retry_at is a no-op: the re-enqueued row survives. + Expect(repo.DeleteIfUnchanged("al", "d1", model.ImageTypePrimary, original)).To(Succeed()) + n, _ := repo.Count() + Expect(n).To(Equal(int64(1))) + + // Deleting with the current retry_at removes it. + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(repo.DeleteIfUnchanged("al", "d1", model.ImageTypePrimary, got[0].RetryAt)).To(Succeed()) + n, _ = repo.Count() + Expect(n).To(BeZero()) + }) + + It("purges queue rows whose entity no longer exists, per kind", func() { + Expect(repo.Enqueue( + item("al", albumSgtPeppers.ID, model.ArtworkPriorityScan), + item("al", "no-such-album", model.ArtworkPriorityScan), + item("ar", artistKraftwerk.ID, model.ArtworkPriorityScan), + item("ar", "no-such-artist", model.ArtworkPriorityScan), + item("pl", plsBest.ID, model.ArtworkPriorityScan), + item("pl", "no-such-playlist", model.ArtworkPriorityScan), + item("ra", radioWithHomePage.ID, model.ArtworkPriorityScan), + item("ra", "no-such-radio", model.ArtworkPriorityScan), + )).To(Succeed()) + + purged, err := repo.PurgeDangling() + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(Equal(int64(4))) + + got, _ := repo.DequeueBatch(100) + ids := make([]string, 0, len(got)) + for _, it := range got { + ids = append(ids, it.ItemID) + } + Expect(ids).To(ConsistOf(albumSgtPeppers.ID, artistKraftwerk.ID, plsBest.ID, radioWithHomePage.ID)) + }) + It("enqueues stale absent states for recheck", func() { awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) old := time.Now().Add(-48 * time.Hour) diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go index f62afe43a..8d01ef9c0 100644 --- a/tests/mock_artwork_queue_repo.go +++ b/tests/mock_artwork_queue_repo.go @@ -13,6 +13,8 @@ type MockArtworkQueueRepo struct { Err error // ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state. ItemArtworkSource *MockArtworkRepo + // ExistingIDs, keyed by item_kind, backs PurgeDangling; a nil per-kind map keeps that kind. + ExistingIDs map[string]map[string]bool } func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo { @@ -90,6 +92,35 @@ func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error { return nil } +func (m *MockArtworkQueueRepo) DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + if it, ok := m.Data[k]; ok && it.RetryAt.Equal(retryAt) { + delete(m.Data, k) + } + return nil +} + +func (m *MockArtworkQueueRepo) PurgeDangling() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + var purged int64 + for k, it := range m.Data { + existing := m.ExistingIDs[it.ItemKind] + if existing == nil { + continue + } + if !existing[it.ItemID] { + delete(m.Data, k) + purged++ + } + } + return purged, nil +} + func (m *MockArtworkQueueRepo) Count() (int64, error) { if m.Err != nil { return 0, m.Err From 0fbbd013573af7f0de41b1279dc9a80d2b4ce3b7 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 11:57:48 -0400 Subject: [PATCH 34/51] style(artwork): fix comment accuracy and budget; fingerprint ArtistImageFolder Correct the inverted workerDeps.extGate comment, trim over-budget doc comments, and add conf.Server.ArtistImageFolder to the resolution fingerprint so an image-folder change re-resolves artist artwork. --- core/artwork/housekeeping.go | 4 ++-- core/artwork/housekeeping_test.go | 7 +++++++ core/artwork/processor.go | 4 ++-- core/artwork/worker_soak_test.go | 6 ++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index 6f1fa20a3..fea80eec6 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -26,8 +26,8 @@ var staleAbsentKinds = []string{"ar", "al", "pl", "ra"} // Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a // change means previously resolved (or absent) state may no longer be correct. func Fingerprint() string { - raw := fmt.Sprintf("%s|%s|%s|%t|%s", - conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, + raw := fmt.Sprintf("%s|%s|%s|%s|%t|%s", + conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, conf.Server.ArtistImageFolder, conf.Server.Agents, conf.Server.EnableExternalServices, consts.Version) sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive return hex.EncodeToString(sum[:]) diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go index 8272514b0..4103d94c7 100644 --- a/core/artwork/housekeeping_test.go +++ b/core/artwork/housekeeping_test.go @@ -72,6 +72,13 @@ var _ = Describe("Housekeeping", func() { f2 := Fingerprint() Expect(f1).NotTo(Equal(f2)) }) + + It("changes when ArtistImageFolder changes", func() { + conf.Server.ArtistImageFolder = "/before" + f1 := Fingerprint() + conf.Server.ArtistImageFolder = "/after" + Expect(Fingerprint()).NotTo(Equal(f1)) + }) }) Describe("Backfill", func() { diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 0c4d6c520..94262f552 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -31,8 +31,8 @@ const ( // thumbnailSize is the max dimension fed to blurhash. const thumbnailSize = 128 -// workerDeps are the collaborators processItem needs; extGate is nil outside -// tests, in which case resolveItem falls back to a plain passthrough. +// workerDeps are the collaborators processItem needs; extGate is set by NewWorker in +// production and nil only in tests, where resolveItem falls back to a plain passthrough. type workerDeps struct { ds model.DataStore store *ImageStore diff --git a/core/artwork/worker_soak_test.go b/core/artwork/worker_soak_test.go index f6ecef422..03ed1247c 100644 --- a/core/artwork/worker_soak_test.go +++ b/core/artwork/worker_soak_test.go @@ -19,10 +19,8 @@ import ( // performance benchmark, so it favors a stable signal over raw speed. const soakCycles = 2200 -// TestWorkerSoak drives processItem across a mix of sources (folder, embedded -// extraction, dangling refs) for many cycles, reading each acquired image back -// through ImageStore.Open, and asserts goroutines/heap plateau instead of -// growing unbounded. Skipped under -short. +// TestWorkerSoak runs processItem over many cycles across a mix of sources, asserting +// goroutines/heap plateau instead of growing unbounded (a leak guard). Skipped under -short. func TestWorkerSoak(t *testing.T) { if testing.Short() { t.Skip("skipping soak test in short mode") From c57496d50d581d5c8e591f5a8d3629516c999f30 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 12:04:17 -0400 Subject: [PATCH 35/51] fix(artwork): treat missing local playlist cover as definitive, not transient A playlist ExternalImageURL pointing at a local file that fails to open was routed through extError, causing failed/48h-retry loops that burn a rate limiter token forever instead of falling through to the generated grid. --- core/artwork/resolve.go | 5 ++++- core/artwork/resolve_test.go | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 272234c9d..af151ef4a 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -278,7 +278,10 @@ func fromPlaylistExternalSource(ctx context.Context, pl model.Playlist) sourceFu } return fromURL(ctx, parsed) } - return fromLocalFile(imgURL)() + // A missing/unreadable local file is a definitive miss, not a transient + // failure to retry: swallow the open error and fall through to the grid. + r, path, _ := fromLocalFile(imgURL)() + return r, path, nil } } diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index cab757dbb..a80d02ca9 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -398,6 +398,20 @@ var _ = Describe("resolveItem", func() { Expect(extGateCalls).To(Equal(1)) }) + It("treats a missing local ExternalImageURL as a definitive miss, not extError", func() { + folderRepo.result = nil // no grid tiles, so the local-file miss is what surfaces + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plm", Name: "Playlist", ExternalImageURL: "/nonexistent/path/cover.jpg"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plm"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeFalse()) + }) + It("yields an empty resolution when no album has art", func() { ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ {ID: "empty1", Name: "Empty"}, From 87095fab08768d02a241311b462807fa7dd8fcc4 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 12:27:55 -0400 Subject: [PATCH 36/51] refactor(artwork): deduplicate purge loop, backfill table, and extGate alias --- core/artwork/housekeeping.go | 67 +++++++++++-------------- core/artwork/processor.go | 2 +- core/artwork/resolve.go | 12 ++--- persistence/artwork_queue_repository.go | 14 +----- persistence/artwork_repository.go | 15 ++++-- 5 files changed, 47 insertions(+), 63 deletions(-) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index fea80eec6..c26b6790d 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -11,6 +11,7 @@ import ( "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/slice" ) // FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against @@ -46,36 +47,36 @@ func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { return false, nil } - artists, err := ds.Artist(ctx).GetAll() - if err != nil { - return false, err + // Artists first: few entities, most external-dependent, so they get queue headstart. + kinds := []struct { + kind string + fetch func() ([]string, error) + }{ + {"ar", func() ([]string, error) { + as, err := ds.Artist(ctx).GetAll() + return slice.Map(as, func(a model.Artist) string { return a.ID }), err + }}, + {"al", func() ([]string, error) { + as, err := ds.Album(ctx).GetAll() + return slice.Map(as, func(a model.Album) string { return a.ID }), err + }}, + {"pl", func() ([]string, error) { + ps, err := ds.Playlist(ctx).GetAll() + return slice.Map(ps, func(p model.Playlist) string { return p.ID }), err + }}, + {"ra", func() ([]string, error) { + rs, err := ds.Radio(ctx).GetAll() + return slice.Map(rs, func(r model.Radio) string { return r.ID }), err + }}, } - if err := enqueueBackfillKind(ctx, ds, "ar", idsOf(artists, func(a model.Artist) string { return a.ID })); err != nil { - return false, err - } - - albums, err := ds.Album(ctx).GetAll() - if err != nil { - return false, err - } - if err := enqueueBackfillKind(ctx, ds, "al", idsOf(albums, func(a model.Album) string { return a.ID })); err != nil { - return false, err - } - - playlists, err := ds.Playlist(ctx).GetAll() - if err != nil { - return false, err - } - if err := enqueueBackfillKind(ctx, ds, "pl", idsOf(playlists, func(p model.Playlist) string { return p.ID })); err != nil { - return false, err - } - - radios, err := ds.Radio(ctx).GetAll() - if err != nil { - return false, err - } - if err := enqueueBackfillKind(ctx, ds, "ra", idsOf(radios, func(r model.Radio) string { return r.ID })); err != nil { - return false, err + for _, k := range kinds { + ids, err := k.fetch() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, k.kind, ids); err != nil { + return false, err + } } if err := props.Put(FingerprintPropertyKey, current); err != nil { @@ -98,14 +99,6 @@ func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, i return ds.ArtworkQueue(ctx).Enqueue(items...) } -func idsOf[T any](items []T, id func(T) string) []string { - ids := make([]string, len(items)) - for i, it := range items { - ids[i] = id(it) - } - return ids -} - // EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across // every artwork-bearing kind, for the periodic recheck job. func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error { diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 94262f552..8852f70b8 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -38,7 +38,7 @@ type workerDeps struct { store *ImageStore prov external.Provider ffmpeg ffmpeg.FFmpeg - extGate func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) + extGate extGateFunc } // processItem resolves one queue item end to end: find an image, hash/decode/ diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index af151ef4a..7bc5115d0 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -39,7 +39,7 @@ func passthroughExtGate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, } // resolveItem walks the kind's priority chain and returns the first hit. -func resolveItem(ctx context.Context, ds model.DataStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg, item model.ArtworkQueueItem, extGate func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error)) (resolution, error) { +func resolveItem(ctx context.Context, ds model.DataStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg, item model.ArtworkQueueItem, extGate extGateFunc) (resolution, error) { if extGate == nil { extGate = passthroughExtGate } @@ -290,12 +290,10 @@ func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, em return resolution{}, false } abs := lib.Abs(embedRel) - mtime := mtimeViaFS(lib.FS, embedRel) - if r, _, _ := fromTag(ctx, lib.FS, embedRel)(); r != nil { - return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtime}, true - } - if r, _, _ := fromFFmpegTag(ctx, ffm, abs)(); r != nil { - return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtime}, true + for _, sf := range []sourceFunc{fromTag(ctx, lib.FS, embedRel), fromFFmpegTag(ctx, ffm, abs)} { + if r, _, _ := sf(); r != nil { + return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtimeViaFS(lib.FS, embedRel)}, true + } } return resolution{}, false } diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go index eb5f21ecd..4a6a68eb1 100644 --- a/persistence/artwork_queue_repository.go +++ b/persistence/artwork_queue_repository.go @@ -78,19 +78,7 @@ func (r *artworkQueueRepository) DeleteIfUnchanged(kind, id, imageType string, r // PurgeDangling removes queue rows whose entity no longer exists, per kind. func (r *artworkQueueRepository) PurgeDangling() (int64, error) { - var total int64 - for kind, table := range danglingItemArtworkKinds { - del := Delete(r.tableName).Where(And{ - Eq{"item_kind": kind}, - Expr("item_id NOT IN (SELECT id FROM " + table + ")"), - }) - c, err := r.executeSQL(del) - if err != nil { - return total, err - } - total += c - } - return total, nil + return purgeDangling(r.executeSQL, r.tableName) } func (r *artworkQueueRepository) Count() (int64, error) { diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 622f47bfe..9ac32b42e 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -122,14 +122,15 @@ var danglingItemArtworkKinds = map[string]string{ "ra": "radio", } -func (r *artworkRepository) PurgeDanglingItemArtwork() (int64, error) { +// purgeDangling deletes rows in table whose owning entity is gone, one statement per kind. +func purgeDangling(execute func(Sqlizer) (int64, error), table string) (int64, error) { var total int64 - for kind, table := range danglingItemArtworkKinds { - del := Delete(itemArtworkTable).Where(And{ + for kind, entityTable := range danglingItemArtworkKinds { + del := Delete(table).Where(And{ Eq{"item_kind": kind}, - Expr("item_id NOT IN (SELECT id FROM " + table + ")"), + Expr("item_id NOT IN (SELECT id FROM " + entityTable + ")"), }) - c, err := r.items.executeSQL(del) + c, err := execute(del) if err != nil { return total, err } @@ -138,6 +139,10 @@ func (r *artworkRepository) PurgeDanglingItemArtwork() (int64, error) { return total, nil } +func (r *artworkRepository) PurgeDanglingItemArtwork() (int64, error) { + return purgeDangling(r.items.executeSQL, itemArtworkTable) +} + func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { sel := Select("*").From(itemArtworkTable). Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) From 67f6d8aee810301e471a09bd7c5c5633266edf13 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 12:38:40 -0400 Subject: [PATCH 37/51] fix(artwork): cap resolved image reads A user-editable ExternalImageURL can point at an arbitrarily large endpoint; a fast server could make the worker buffer hundreds of MB inside the 5s HTTP timeout. Bound the read to a fixed 20MB cap (no config knob) via io.LimitReader and fail the item if it is exceeded. --- core/artwork/processor.go | 10 +++++++++- core/artwork/processor_test.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 8852f70b8..262d06518 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -31,6 +31,10 @@ const ( // thumbnailSize is the max dimension fed to blurhash. const thumbnailSize = 128 +// maxImageBytes caps a resolved image read: a user-editable ExternalImageURL could +// point at an arbitrarily large endpoint, and 20MB is generous for any real cover. +const maxImageBytes = 20 << 20 + // workerDeps are the collaborators processItem needs; extGate is set by NewWorker in // production and nil only in tests, where resolveItem falls back to a plain passthrough. type workerDeps struct { @@ -60,11 +64,15 @@ func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueI } defer res.reader.Close() - data, err := io.ReadAll(res.reader) + data, err := io.ReadAll(io.LimitReader(res.reader, maxImageBytes+1)) if err != nil { log.Warn(ctx, "artwork: failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, err) return outcomeFailed } + if len(data) > maxImageBytes { + log.Warn(ctx, "artwork: resolved image exceeds size cap", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "cap", maxImageBytes) + return outcomeFailed + } log.Debug(ctx, "artwork: read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "bytes", len(data)) hash, err := HashImage(bytes.NewReader(data)) diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go index a73cd4fca..64b0181bd 100644 --- a/core/artwork/processor_test.go +++ b/core/artwork/processor_test.go @@ -186,6 +186,27 @@ var _ = Describe("processItem", func() { Expect(err).To(MatchError(model.ErrNotFound)) }) + It("oversized read: a resolved image larger than the cap fails without writing state", func() { + tmpDir := GinkgoT().TempDir() + conf.Server.DataFolder = conf.NewDir(tmpDir) + Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) + imgPath := filepath.Join(tmpDir, "artwork", "radio", "big_test.jpg") + f, err := os.Create(imgPath) + Expect(err).ToNot(HaveOccurred()) + Expect(f.Truncate(maxImageBytes + 1)).To(Succeed()) + Expect(f.Close()).To(Succeed()) + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.Data = map[string]*model.Radio{"big": {ID: "big", Name: "Radio", UploadedImage: "big_test.jpg"}} + ds.MockedRadio = radioRepo + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "big"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err = artRepo.GetItemArtwork("ra", "big", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + It("store write failure: fails without writing state", func() { ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ {ID: "al7", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, From 6afcb93a9b84889ea08f4c27f3948e395c765e70 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 12:39:15 -0400 Subject: [PATCH 38/51] fix(artwork): retry higher-priority external art after fallback hit With CoverArtPriority="external,cover.jpg", a transient external failure followed by a folder hit dropped the external error: the worker recorded found and deleted the queue row, so the configured higher-priority external art was never retried. Carry extError onto the fallback resolution and add an outcomeFoundStale that persists+serves the art but reschedules via MarkFailed, giving the external source another chance. When external later answers definitively-not-found, the hit is not stale and the row is deleted. --- core/artwork/processor.go | 6 +++++ core/artwork/processor_test.go | 22 +++++++++++++++++++ core/artwork/resolve.go | 11 ++++++++-- core/artwork/resolve_test.go | 40 ++++++++++++++++++++++++++++++++++ core/artwork/worker.go | 2 +- core/artwork/worker_test.go | 28 ++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 3 deletions(-) diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 262d06518..73a522daf 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -24,6 +24,9 @@ type outcome int const ( outcomeFound outcome = iota + // outcomeFoundStale: state was written and is served, but a higher-priority external + // step failed, so the row must retry (via MarkFailed) to give that source another chance. + outcomeFoundStale outcomeAbsent outcomeFailed ) @@ -116,6 +119,9 @@ func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueI log.Warn(ctx, "artwork: failed to persist item artwork state", "kind", item.ItemKind, "id", item.ItemID, err) return outcomeFailed } + if res.extError { + return outcomeFoundStale + } return outcomeFound } diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go index 64b0181bd..876124bc3 100644 --- a/core/artwork/processor_test.go +++ b/core/artwork/processor_test.go @@ -136,6 +136,28 @@ var _ = Describe("processItem", func() { Expect(err).To(MatchError(model.ErrNotFound)) }) + It("found-stale: a fallback hit after a transient external failure persists state and returns outcomeFoundStale", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "alstale", Name: "Album", FolderIDs: []string{"f1"}}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"}) + Expect(out).To(Equal(outcomeFoundStale)) + + ia, err := artRepo.GetItemArtwork("al", "alstale", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).ToNot(BeEmpty()) + Expect(ia.Source).To(Equal("folder")) + }) + It("dedup: a second item with identical bytes skips decode and reuses the artwork row", func() { folderRepo.result = []model.Folder{{ Path: "tests/fixtures/artist/an-album", diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 7bc5115d0..bb2a3f01b 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -27,7 +27,9 @@ type resolution struct { source string // model.ItemArtwork.Source value: "folder", "embedded", "external", "upload", "generated" sourcePath string // backing library/upload file (folder/upload: the image; embedded: the audio file); "" otherwise refMtime int64 // mtime of sourcePath at resolution time; 0 when no sourcePath - extError bool // an external source errored/timed out (forces failed, never absent) + // external source errored/timed out. With no reader: forces failed (never absent). + // On a hit: a higher-priority external step failed—serve this, but retry later. + extError bool } // extGateFunc is an alias for the external-step wrapper the worker injects (rate @@ -79,6 +81,7 @@ func resolveAlbum(ctx context.Context, ds model.DataStore, prov external.Provide switch { case pattern == "embedded": if res, ok := resolveEmbedded(ctx, lib, ffm, al.EmbedArtPath); ok { + res.extError = extErr return res, nil } case pattern == "external": @@ -89,6 +92,7 @@ func resolveAlbum(ctx context.Context, ds model.DataStore, prov external.Provide } case len(imgFiles) > 0: if res, ok := resolveFolderFile(ctx, lib, imgFiles, pattern); ok { + res.extError = extErr return res, nil } } @@ -145,6 +149,7 @@ func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provid } case pattern == "image-folder": if res, ok := resolveArtistImageFolder(ar); ok { + res.extError = extErr return res, nil } case strings.HasPrefix(pattern, "album/"): @@ -152,6 +157,7 @@ func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provid continue } if res, ok := resolveFolderFile(ctx, lib, imgFiles, strings.TrimPrefix(pattern, "album/")); ok { + res.extError = extErr return res, nil } default: @@ -159,6 +165,7 @@ func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provid continue } if res, ok := resolveArtistFolderPattern(ctx, lib, artistFolder, pattern); ok { + res.extError = extErr return res, nil } } @@ -236,7 +243,7 @@ func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Prov if err != nil { return resolution{extError: extErr}, nil //nolint:nilerr // encode failure is a soft "no image", not a resolveItem error } - return resolution{reader: r, source: "generated"}, nil + return resolution{reader: r, source: "generated", extError: extErr}, nil } // resolveRadio ports reader_radio.go: only an uploaded image, no fallback. diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index a80d02ca9..21b742345 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -148,6 +148,46 @@ var _ = Describe("resolveItem", func() { Expect(res.extError).To(BeFalse()) }) + It("carries extError onto a fallback folder hit after a transient external failure", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al6", Name: "Album", FolderIDs: []string{"f1"}}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(res.extError).To(BeTrue()) + }) + + It("does not carry extError onto a fallback folder hit after a definitive external not-found", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "al7", Name: "Album", FolderIDs: []string{"f1"}}, + }) + // prov.albumImage left nil -> fakeExternalProvider returns model.ErrNotFound + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(res.extError).To(BeFalse()) + }) + It("routes the external step through a custom extGate", func() { conf.Server.CoverArtPriority = "external" ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ diff --git a/core/artwork/worker.go b/core/artwork/worker.go index fb20a2bbc..bc3e47af0 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -154,7 +154,7 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) { if err := queue.DeleteIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt); err != nil { log.Warn(ctx, "artwork: could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err) } - case outcomeFailed: + case outcomeFoundStale, outcomeFailed: retryAt := time.Now().Add(backoff(item.Attempts)) if err := queue.MarkFailed(item.ItemKind, item.ItemID, item.ImageType, retryAt); err != nil { log.Warn(ctx, "artwork: could not reschedule failed queue item", "kind", item.ItemKind, "id", item.ItemID, err) diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index 058d707b8..7865b6cfc 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -135,6 +135,34 @@ var _ = Describe("Worker", func() { Expect(err).To(MatchError(model.ErrNotFound), "a timeout must never settle on absent") }) + It("reschedules a found-stale item via MarkFailed while keeping its served state", func() { + conf.Server.CoverArtPriority = "external, cover.jpg" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "alstale", Name: "Album", FolderIDs: []string{"f1"}}, + }) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"})).To(Succeed()) + + n, err := w.drain(ctx, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + it := findQueued(queueRepo, "al", "alstale") + Expect(it).ToNot(BeNil(), "a found-stale row must survive for a higher-priority retry") + Expect(it.Attempts).To(Equal(1)) + Expect(it.RetryAt).To(BeTemporally(">", time.Now())) + + ia, err := artRepo.GetItemArtwork("al", "alstale", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder"), "the fallback art is served meanwhile") + }) + It("keeps a row re-enqueued between dequeue and delete", func() { folderRepo.result = []model.Folder{{ Path: "tests/fixtures/artist/an-album", From 5482784bfc19a5a8a0aae74623d25d9ea0eaa602 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 12:39:16 -0400 Subject: [PATCH 39/51] fix(artwork): treat playlist cover URL 404 as definitive miss The playlist ExternalImageURL step used sources.go's fromURL, which maps any non-200 to a generic error, so a stale URL returning 404/410 was classified transient: infinite backoff plus it counted toward the circuit breaker, blocking valid external work. Add a local fetch in resolve.go that maps 404/410 to model.ErrNotFound (definitive) while keeping other non-200s transient. sources.go is left untouched. --- core/artwork/resolve.go | 26 ++++++++++++++++++++++- core/artwork/resolve_test.go | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index bb2a3f01b..8597ec836 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -10,12 +10,15 @@ import ( "image/png" "io" "io/fs" + "net/http" "net/url" "os" "strings" + "time" "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/core/external" "github.com/navidrome/navidrome/core/ffmpeg" "github.com/navidrome/navidrome/model" @@ -283,7 +286,7 @@ func fromPlaylistExternalSource(ctx context.Context, pl model.Playlist) sourceFu if !conf.Server.EnableM3UExternalAlbumArt { return nil, "", nil } - return fromURL(ctx, parsed) + return fetchPlaylistImageURL(ctx, parsed) } // A missing/unreadable local file is a definitive miss, not a transient // failure to retry: swallow the open error and fall through to the grid. @@ -292,6 +295,27 @@ func fromPlaylistExternalSource(ctx context.Context, pl model.Playlist) sourceFu } } +// Like sources.go's fromURL but maps 404/410 to ErrNotFound (definitive), so a stale M3U +// cover URL falls through to the grid instead of retrying forever and tripping the breaker. +func fetchPlaylistImageURL(ctx context.Context, imageURL *url.URL) (io.ReadCloser, string, error) { + hc := http.Client{Timeout: 5 * time.Second} + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, imageURL.String(), nil) + req.Header.Set("User-Agent", consts.HTTPUserAgent) + resp, err := hc.Do(req) //nolint:gosec + if err != nil { + return nil, "", err + } + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + resp.Body.Close() + return nil, "", model.ErrNotFound + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, "", fmt.Errorf("error retrieving artwork from %s: %s", imageURL, resp.Status) + } + return resp.Body, imageURL.String(), nil +} + func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, embedRel string) (resolution, bool) { if embedRel == "" { return resolution{}, false diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index 21b742345..96702ef33 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -5,6 +5,8 @@ import ( "errors" "image" "io" + "net/http" + "net/http/httptest" "net/url" "os" "path/filepath" @@ -452,6 +454,45 @@ var _ = Describe("resolveItem", func() { Expect(res.extError).To(BeFalse()) }) + It("treats an ExternalImageURL 404 as a definitive miss and falls through to the grid", func() { + conf.Server.EnableM3UExternalAlbumArt = true + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl404", Name: "Playlist", ExternalImageURL: srv.URL}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl404"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("generated")) + Expect(res.extError).To(BeFalse()) + }) + + It("treats an ExternalImageURL 500 as a transient failure and sets extError", func() { + conf.Server.EnableM3UExternalAlbumArt = true + folderRepo.result = nil // no grid tiles, so the external failure is what surfaces + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "pl500", Name: "Playlist", ExternalImageURL: srv.URL}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl500"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.extError).To(BeTrue()) + }) + It("yields an empty resolution when no album has art", func() { ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ {ID: "empty1", Name: "Empty"}, From 3d32157403807751eed58ee1106190c10386c10c Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 13:03:38 -0400 Subject: [PATCH 40/51] test(artwork): move soak test into the Ginkgo suite --- core/artwork/worker_soak_test.go | 261 +++++++++++++++---------------- 1 file changed, 127 insertions(+), 134 deletions(-) diff --git a/core/artwork/worker_soak_test.go b/core/artwork/worker_soak_test.go index 03ed1247c..355ecb7ca 100644 --- a/core/artwork/worker_soak_test.go +++ b/core/artwork/worker_soak_test.go @@ -10,148 +10,141 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/conf/configtest" - "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) // soakCycles is deliberately >2000: this is a leak regression guard, not a // performance benchmark, so it favors a stable signal over raw speed. const soakCycles = 2200 -// TestWorkerSoak runs processItem over many cycles across a mix of sources, asserting -// goroutines/heap plateau instead of growing unbounded (a leak guard). Skipped under -short. -func TestWorkerSoak(t *testing.T) { - if testing.Short() { - t.Skip("skipping soak test in short mode") - } - tests.Init(t, false) - log.SetLevel(log.LevelFatal) - defer configtest.SetupConfig()() +var _ = Describe("Worker soak", func() { + // Runs processItem over many cycles across a mix of sources, asserting + // goroutines/heap plateau instead of growing unbounded (a leak guard). Skipped under -short. + It("does not leak goroutines, heap, or fds over many acquisition cycles", func() { + if testing.Short() { + Skip("skipping soak test in short mode") + } + DeferCleanup(configtest.SetupConfig()) - repoRoot, err := os.Getwd() - if err != nil { - t.Fatal(err) - } + repoRoot, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) - libRepo := &tests.MockLibraryRepo{} - libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) - folderRepo := &fakeFolderRepo{result: []model.Folder{{ - Path: "tests/fixtures/artist/an-album", - ImageFiles: []string{"cover.jpg"}, - }}} - ffm := tests.NewMockFFmpeg("") - prov := &fakeExternalProvider{} - artRepo := tests.CreateMockArtworkRepo() - albumRepo := tests.CreateMockAlbumRepo() - albumRepo.SetData(model.Albums{ - {ID: "al-folder", Name: "Folder Album", FolderIDs: []string{"f1"}}, - {ID: "al-embed", Name: "Embedded Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + folderRepo := &fakeFolderRepo{result: []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }}} + ffm := tests.NewMockFFmpeg("") + prov := &fakeExternalProvider{} + artRepo := tests.CreateMockArtworkRepo() + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{ + {ID: "al-folder", Name: "Folder Album", FolderIDs: []string{"f1"}}, + {ID: "al-embed", Name: "Embedded Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + ds := &tests.MockDataStore{ + MockedFolder: folderRepo, + MockedLibrary: libRepo, + MockedArtwork: artRepo, + MockedAlbum: albumRepo, + } + store := NewImageStore(GinkgoT().TempDir()) + deps := &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm} + conf.Server.CoverArtPriority = "cover.jpg, embedded" + + // Dangling refs (al/ra ids the repos don't know about) mirror an entity + // deleted after being enqueued; ds.Radio auto-provisions an empty mock repo. + items := []model.ArtworkQueueItem{ + {ItemKind: "al", ItemID: "al-folder"}, + {ItemKind: "al", ItemID: "al-embed"}, + {ItemKind: "al", ItemID: "al-does-not-exist"}, + {ItemKind: "ra", ItemID: "ra-does-not-exist"}, + } + + fdCount := func() int { + if runtime.GOOS != "linux" { + return -1 + } + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + return -1 + } + return len(entries) + } + + settleGoroutines := func() int { + // Background goroutines (GC workers, etc.) can take a moment to wind down; + // poll for two consecutive equal samples instead of trusting a single one. + prev := -1 + for range 100 { + runtime.GC() + n := runtime.NumGoroutine() + if n == prev { + return n + } + prev = n + time.Sleep(10 * time.Millisecond) + } + return prev + } + + baselineGoroutines := settleGoroutines() + baselineFDs := fdCount() + + var heapAt10Pct uint64 + start := time.Now() + for i := range soakCycles { + it := items[i%len(items)] + out := processItem(context.Background(), deps, it) + + // "Serve-adjacent" read-back: exercise the Phase 2 surfaces a caller would + // use after acquisition, not the old serving pipeline. + if out == outcomeFound { + ia, err := artRepo.GetItemArtwork(it.ItemKind, it.ItemID, model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred(), "cycle %d: GetItemArtwork", i) + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred(), "cycle %d: GetImage", i) + rc, err := store.Open(ia.Hash, art.Mime) + switch { + case err == nil: + _, _ = io.Copy(io.Discard, rc) + rc.Close() + case os.IsNotExist(err): + // Folder-backed art has no store file; that's expected. + default: + Expect(err).ToNot(HaveOccurred(), "cycle %d: store.Open", i) + } + } + + if i == soakCycles/10 { + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + heapAt10Pct = ms.HeapAlloc + } + } + elapsed := time.Since(start) + + finalGoroutines := settleGoroutines() + finalFDs := fdCount() + + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + + GinkgoWriter.Printf("soak: cycles=%d elapsed=%s goroutines(baseline=%d final=%d) heap(10%%-mark=%d final=%d) fds(baseline=%d final=%d)\n", + soakCycles, elapsed, baselineGoroutines, finalGoroutines, heapAt10Pct, ms.HeapAlloc, baselineFDs, finalFDs) + + Expect(finalGoroutines).To(BeNumerically("<=", baselineGoroutines), "goroutine count grew: baseline=%d final=%d", baselineGoroutines, finalGoroutines) + if heapAt10Pct > 0 { + Expect(ms.HeapAlloc).To(BeNumerically("<=", 2*heapAt10Pct), "heap did not plateau: 10%%-mark=%d final=%d (final > 2x 10%%-mark)", heapAt10Pct, ms.HeapAlloc) + } + if runtime.GOOS == "linux" && baselineFDs >= 0 { + Expect(finalFDs).To(BeNumerically("<=", baselineFDs), "fd count grew: baseline=%d final=%d", baselineFDs, finalFDs) + } }) - ds := &tests.MockDataStore{ - MockedFolder: folderRepo, - MockedLibrary: libRepo, - MockedArtwork: artRepo, - MockedAlbum: albumRepo, - } - store := NewImageStore(t.TempDir()) - deps := &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm} - conf.Server.CoverArtPriority = "cover.jpg, embedded" - - // Dangling refs (al/ra ids the repos don't know about) mirror an entity - // deleted after being enqueued; ds.Radio auto-provisions an empty mock repo. - items := []model.ArtworkQueueItem{ - {ItemKind: "al", ItemID: "al-folder"}, - {ItemKind: "al", ItemID: "al-embed"}, - {ItemKind: "al", ItemID: "al-does-not-exist"}, - {ItemKind: "ra", ItemID: "ra-does-not-exist"}, - } - - fdCount := func() int { - if runtime.GOOS != "linux" { - return -1 - } - entries, err := os.ReadDir("/proc/self/fd") - if err != nil { - return -1 - } - return len(entries) - } - - settleGoroutines := func() int { - // Background goroutines (GC workers, etc.) can take a moment to wind down; - // poll for two consecutive equal samples instead of trusting a single one. - prev := -1 - for range 100 { - runtime.GC() - n := runtime.NumGoroutine() - if n == prev { - return n - } - prev = n - time.Sleep(10 * time.Millisecond) - } - return prev - } - - baselineGoroutines := settleGoroutines() - baselineFDs := fdCount() - - var heapAt10Pct uint64 - start := time.Now() - for i := range soakCycles { - it := items[i%len(items)] - out := processItem(context.Background(), deps, it) - - // "Serve-adjacent" read-back: exercise the Phase 2 surfaces a caller would - // use after acquisition, not the old serving pipeline. - if out == outcomeFound { - ia, err := artRepo.GetItemArtwork(it.ItemKind, it.ItemID, model.ImageTypePrimary) - if err != nil { - t.Fatalf("cycle %d: GetItemArtwork: %v", i, err) - } - art, err := artRepo.GetImage(ia.Hash) - if err != nil { - t.Fatalf("cycle %d: GetImage: %v", i, err) - } - rc, err := store.Open(ia.Hash, art.Mime) - switch { - case err == nil: - _, _ = io.Copy(io.Discard, rc) - rc.Close() - case os.IsNotExist(err): - // Folder-backed art has no store file; that's expected. - default: - t.Fatalf("cycle %d: store.Open: %v", i, err) - } - } - - if i == soakCycles/10 { - runtime.GC() - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - heapAt10Pct = ms.HeapAlloc - } - } - elapsed := time.Since(start) - - finalGoroutines := settleGoroutines() - finalFDs := fdCount() - - runtime.GC() - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - - t.Logf("soak: cycles=%d elapsed=%s goroutines(baseline=%d final=%d) heap(10%%-mark=%d final=%d) fds(baseline=%d final=%d)", - soakCycles, elapsed, baselineGoroutines, finalGoroutines, heapAt10Pct, ms.HeapAlloc, baselineFDs, finalFDs) - - if finalGoroutines > baselineGoroutines { - t.Errorf("goroutine count grew: baseline=%d final=%d", baselineGoroutines, finalGoroutines) - } - if heapAt10Pct > 0 && ms.HeapAlloc > 2*heapAt10Pct { - t.Errorf("heap did not plateau: 10%%-mark=%d final=%d (final > 2x 10%%-mark)", heapAt10Pct, ms.HeapAlloc) - } - if runtime.GOOS == "linux" && baselineFDs >= 0 && finalFDs > baselineFDs { - t.Errorf("fd count grew: baseline=%d final=%d", baselineFDs, finalFDs) - } -} +}) From b3526c0fbad4af15a6b9a7eee854cfcb55227895 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 13:15:47 -0400 Subject: [PATCH 41/51] test(artwork): make leak and permission tests pass on linux goleak now ignores notify's nonrecursive-tree goroutines (linux uses inotify, which spawns dispatch+internal instead of darwin's recursive dispatch), and the read-only-dir prune spec skips under root, where permission bits cannot make Remove fail. --- core/artwork/artwork_suite_test.go | 6 ++++-- core/artwork/prune_test.go | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/artwork/artwork_suite_test.go b/core/artwork/artwork_suite_test.go index b721200e1..e2a1a591c 100644 --- a/core/artwork/artwork_suite_test.go +++ b/core/artwork/artwork_suite_test.go @@ -23,9 +23,11 @@ func TestArtwork(t *testing.T) { // package's control, so they're ignored by exact top-function instead. defer goleak.VerifyNone(t, goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"), - // notify's own init() starts this dispatcher the moment it's imported - // (via core/storage/local or plugins); it never exits. + // notify's own init() starts a singleton tree the moment it's imported (via + // core/storage/local or plugins); recursive on darwin, nonrecursive on linux. goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*recursiveTree).dispatch"), + goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*nonrecursiveTree).dispatch"), + goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*nonrecursiveTree).internal"), // The old cache_warmer.go starts a goroutine per NewCacheWarmer call with // no shutdown path (dark-launch target for Phase 2, not touched here). goleak.IgnoreTopFunction("github.com/navidrome/navidrome/core/artwork.(*cacheWarmer).waitSignal"), diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go index 5511b2305..646ad7440 100644 --- a/core/artwork/prune_test.go +++ b/core/artwork/prune_test.go @@ -189,6 +189,9 @@ var _ = Describe("Prune", func() { It("warns and continues past a store.Remove failure instead of aborting the loop", func() { tests.SkipOnWindows("uses Unix file permission bits") + if os.Geteuid() == 0 { + Skip("read-only dir cannot block root (e.g. tests in a container)") + } old := time.Now().Add(-2 * time.Hour) blocked := []byte("blocked-bytes") From d6434b99293d427f503da3342b56cd7aa09db469 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 14:02:48 -0400 Subject: [PATCH 42/51] fix(artwork): reject decompression-bomb dimensions before decoding --- core/artwork/processor.go | 7 +++++++ core/artwork/processor_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 73a522daf..58def5fc0 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -38,6 +38,10 @@ const thumbnailSize = 128 // point at an arbitrarily large endpoint, and 20MB is generous for any real cover. const maxImageBytes = 20 << 20 +// maxImagePixels caps declared dimensions: a tiny compressed file can declare a +// huge canvas that image.Decode would expand into gigabytes (decompression bomb). +const maxImagePixels = 64 << 20 + // workerDeps are the collaborators processItem needs; extGate is set by NewWorker in // production and nil only in tests, where resolveItem falls back to a plain passthrough. type workerDeps struct { @@ -147,6 +151,9 @@ func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwor if err != nil { return nil, fmt.Errorf("decode image config: %w", err) } + if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels { + return nil, fmt.Errorf("image dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels) + } img, _, err := image.Decode(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("decode image: %w", err) diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go index 876124bc3..574bb106b 100644 --- a/core/artwork/processor_test.go +++ b/core/artwork/processor_test.go @@ -2,7 +2,9 @@ package artwork import ( "context" + "encoding/binary" "errors" + "hash/crc32" "net/url" "os" "path/filepath" @@ -16,6 +18,21 @@ import ( . "github.com/onsi/gomega" ) +// pngHeaderWithDims builds just a PNG signature + IHDR chunk declaring w×h. DecodeConfig +// reads the header without touching pixel data, so the body can be omitted entirely. +func pngHeaderWithDims(w, h uint32) []byte { + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:], w) + binary.BigEndian.PutUint32(ihdr[4:], h) + ihdr[8] = 8 // bit depth + ihdr[9] = 2 // color type: truecolor + chunk := append([]byte("IHDR"), ihdr...) + out := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a} + out = binary.BigEndian.AppendUint32(out, uint32(len(ihdr))) + out = append(out, chunk...) + return binary.BigEndian.AppendUint32(out, crc32.ChecksumIEEE(chunk)) +} + var _ = Describe("processItem", func() { var ( ctx context.Context @@ -229,6 +246,13 @@ var _ = Describe("processItem", func() { Expect(err).To(MatchError(model.ErrNotFound)) }) + It("decompression bomb: rejects huge declared dimensions before the full decode", func() { + data := pngHeaderWithDims(50000, 50000) // 2.5 gigapixels, far above the cap + _, err := decodeArtwork(ctx, "bomb", data) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dimensions")) + }) + It("store write failure: fails without writing state", func() { ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ {ID: "al7", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, From bc30ce67c6ed6c1ca72177b1843d319f28e35d9b Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 14:02:52 -0400 Subject: [PATCH 43/51] fix(artwork): keep fresh re-enqueues ahead of stale failure backoff --- core/artwork/worker.go | 4 ++- core/artwork/worker_test.go | 24 +++++++++++++++++ model/artwork.go | 3 +++ persistence/artwork_queue_repository.go | 11 ++++++++ persistence/artwork_queue_repository_test.go | 28 ++++++++++++++++++++ tests/mock_artwork_queue_repo.go | 13 +++++++++ 6 files changed, 82 insertions(+), 1 deletion(-) diff --git a/core/artwork/worker.go b/core/artwork/worker.go index bc3e47af0..4be32fee8 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -155,8 +155,10 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) { log.Warn(ctx, "artwork: could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err) } case outcomeFoundStale, outcomeFailed: + // MarkFailedIfUnchanged, not MarkFailed: a scan that re-enqueued this row mid-flight reset + // retry_at, so stale backoff must not stomp its fresh, immediate eligibility. retryAt := time.Now().Add(backoff(item.Attempts)) - if err := queue.MarkFailed(item.ItemKind, item.ItemID, item.ImageType, retryAt); err != nil { + if err := queue.MarkFailedIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt, retryAt); err != nil { log.Warn(ctx, "artwork: could not reschedule failed queue item", "kind", item.ItemKind, "id", item.ItemID, err) } } diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index 7865b6cfc..1fe904e21 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -189,6 +189,30 @@ var _ = Describe("Worker", func() { Expect(ia.Source).To(Equal("folder")) }) + It("keeps a fresh re-enqueue ahead of a stale failure backoff", func() { + conf.Server.CoverArtPriority = "external" + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al8", Name: "Album"}}) + prov.albumImage = func(context.Context, string) (*url.URL, error) { + return nil, errors.New("agent timed out") + } + racing := &reenqueueOnDequeue{MockArtworkQueueRepo: queueRepo} + ds.MockedArtworkQueue = racing + w = NewWorker(ds, store, prov, ffm) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al8"})).To(Succeed()) + dequeued := findQueued(queueRepo, "al", "al8").RetryAt + + n, err := w.drain(ctx, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + // The concurrent re-enqueue reset retry_at; the failure path must not stomp it + // with stale backoff nor bump attempts, so the row stays immediately eligible. + it := findQueued(queueRepo, "al", "al8") + Expect(it).ToNot(BeNil()) + Expect(it.Attempts).To(BeZero()) + Expect(it.RetryAt).To(BeTemporally("==", dequeued.Add(time.Minute))) + }) + It("returns zero when the queue is empty", func() { n, err := w.drain(ctx, 2) Expect(err).ToNot(HaveOccurred()) diff --git a/model/artwork.go b/model/artwork.go index 0aec6489a..8db11ce69 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -88,6 +88,9 @@ type ArtworkQueueRepository interface { DequeueBatch(n int) ([]ArtworkQueueItem, error) // MarkFailed increments attempts and pushes retry_at into the future. MarkFailed(kind, id, imageType string, retryAt time.Time) error + // MarkFailedIfUnchanged applies the failure backoff only while retry_at still matches + // seenRetryAt; a concurrent re-enqueue (which resets retry_at) keeps its fresh eligibility. + MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error Delete(kind, id, imageType string) error // DeleteIfUnchanged deletes the row only if its retry_at still matches retryAt, so a // concurrent re-enqueue (which resets retry_at) survives instead of being erased. diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go index 4a6a68eb1..59e130204 100644 --- a/persistence/artwork_queue_repository.go +++ b/persistence/artwork_queue_repository.go @@ -66,6 +66,17 @@ func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt return err } +// MarkFailedIfUnchanged applies the backoff only while retry_at still equals seenRetryAt; +// a concurrent Enqueue resets retry_at, so its fresh eligibility survives untouched. +func (r *artworkQueueRepository) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error { + upd := Update(r.tableName). + Set("attempts", Expr("attempts + 1")). + Set("retry_at", retryAt). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": seenRetryAt}) + _, err := r.executeSQL(upd) + return err +} + func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) } diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go index e96802284..0a6ddefdc 100644 --- a/persistence/artwork_queue_repository_test.go +++ b/persistence/artwork_queue_repository_test.go @@ -54,6 +54,34 @@ var _ = Describe("ArtworkQueueRepository", func() { Expect(got[0].Attempts).To(Equal(2)) }) + It("MarkFailedIfUnchanged applies backoff only while retry_at is unchanged", func() { + Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed()) + // Anchor retry_at in the past (attempts -> 1) so it can never collide with the re-enqueue's now. + Expect(repo.MarkFailed("al", "m1", model.ImageTypePrimary, time.Now().Add(-time.Hour))).To(Succeed()) + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(1)) + original := got[0].RetryAt + + // A concurrent scan re-enqueues, resetting retry_at to now. + Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed()) + + // Failing with the stale retry_at is a no-op: the re-enqueued row keeps its fresh state. + future := time.Now().Add(48 * time.Hour) + Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, original, future)).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1), "the fresh re-enqueue stays immediately eligible") + Expect(got[0].Attempts).To(Equal(1), "the stale failure must not bump attempts") + current := got[0].RetryAt + + // Failing with the current retry_at applies the backoff and bumps attempts. + Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, current, future)).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(BeEmpty(), "backed-off row is hidden until the future retry_at") + all, _ := repo.Count() + Expect(all).To(Equal(int64(1))) + }) + It("deletes on completion and counts", func() { Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed()) n, _ := repo.Count() diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go index 8d01ef9c0..421e0dd15 100644 --- a/tests/mock_artwork_queue_repo.go +++ b/tests/mock_artwork_queue_repo.go @@ -84,6 +84,19 @@ func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt ti return nil } +func (m *MockArtworkQueueRepo) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + if it, ok := m.Data[k]; ok && it.RetryAt.Equal(seenRetryAt) { + it.Attempts++ + it.RetryAt = retryAt + m.Data[k] = it + } + return nil +} + func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error { if m.Err != nil { return m.Err From 7713a6d6b291a4a603f82daa72cd69b6054b11f0 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 14:02:56 -0400 Subject: [PATCH 44/51] fix(artwork): include M3U external art flag in the config fingerprint --- core/artwork/housekeeping.go | 4 ++-- core/artwork/housekeeping_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index c26b6790d..1b67456bf 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -27,9 +27,9 @@ var staleAbsentKinds = []string{"ar", "al", "pl", "ra"} // Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a // change means previously resolved (or absent) state may no longer be correct. func Fingerprint() string { - raw := fmt.Sprintf("%s|%s|%s|%s|%t|%s", + raw := fmt.Sprintf("%s|%s|%s|%s|%t|%t|%s", conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, conf.Server.ArtistImageFolder, - conf.Server.Agents, conf.Server.EnableExternalServices, consts.Version) + conf.Server.Agents, conf.Server.EnableExternalServices, conf.Server.EnableM3UExternalAlbumArt, consts.Version) sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive return hex.EncodeToString(sum[:]) } diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go index 4103d94c7..83ba014c2 100644 --- a/core/artwork/housekeeping_test.go +++ b/core/artwork/housekeeping_test.go @@ -79,6 +79,13 @@ var _ = Describe("Housekeeping", func() { conf.Server.ArtistImageFolder = "/after" Expect(Fingerprint()).NotTo(Equal(f1)) }) + + It("changes when EnableM3UExternalAlbumArt is toggled", func() { + conf.Server.EnableM3UExternalAlbumArt = false + f1 := Fingerprint() + conf.Server.EnableM3UExternalAlbumArt = true + Expect(Fingerprint()).NotTo(Equal(f1)) + }) }) Describe("Backfill", func() { From 55608b2d20693f7db981e4bc6793e4e04cd3e5d4 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 14:03:42 -0400 Subject: [PATCH 45/51] fix(artwork): resolve private playlists with an admin context --- core/artwork/housekeeping.go | 12 ++++++++++ core/artwork/housekeeping_test.go | 38 +++++++++++++++++++++++++++++++ core/artwork/worker.go | 3 +++ core/artwork/worker_test.go | 21 +++++++++++++++++ 4 files changed, 74 insertions(+) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index 1b67456bf..6d4cf4f43 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -11,6 +11,7 @@ import ( "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/utils/slice" ) @@ -34,9 +35,20 @@ func Fingerprint() string { return hex.EncodeToString(sum[:]) } +// withAdminUser wraps ctx with the first admin so repos that apply a per-user visibility +// filter (playlists) expose private rows during this headless work; no admin yet -> unchanged. +func withAdminUser(ctx context.Context, ds model.DataStore) context.Context { + admin, err := ds.User(ctx).FindFirstAdmin() + if err != nil || admin == nil || admin.ID == "" { + return ctx + } + return request.WithUser(ctx, *admin) +} + // Backfill enqueues artwork resolution for every entity when the config fingerprint changed // (or was never stored), artists first so those pages resolve before the larger backlog. func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { + ctx = withAdminUser(ctx, ds) current := Fingerprint() props := ds.Property(ctx) stored, err := props.DefaultGet(FingerprintPropertyKey, "") diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go index 83ba014c2..1df8312e8 100644 --- a/core/artwork/housekeeping_test.go +++ b/core/artwork/housekeeping_test.go @@ -7,11 +7,35 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/conf/configtest" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +// visibilityPlaylistDS models playlist_repository's userFilter: a private playlist is only +// visible when the ctx carries an admin, so headless work must wrap ctx with one first. +type visibilityPlaylistDS struct { + *tests.MockDataStore + private model.Playlist + tracks model.PlaylistTrackRepository +} + +func (v *visibilityPlaylistDS) Playlist(ctx context.Context) model.PlaylistRepository { + repo := tests.CreateMockPlaylistRepo() + repo.TracksRepo = v.tracks + if u, ok := request.UserFrom(ctx); ok && u.IsAdmin { + repo.SetData(model.Playlists{v.private}) + } + return repo +} + +func adminUserRepo() *tests.MockedUserRepo { + repo := tests.CreateMockUserRepo() + Expect(repo.Put(&model.User{ID: "admin", UserName: "admin", IsAdmin: true})).To(Succeed()) + return repo +} + // orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can // assert phase ordering (artists-first) that same-priority timestamps can't guarantee. type orderTrackingQueueRepo struct { @@ -118,6 +142,20 @@ var _ = Describe("Housekeeping", func() { Expect(stored).To(Equal(Fingerprint())) }) + It("enqueues a private playlist by resolving it under an admin context", func() { + ds.MockedUser = adminUserRepo() + vds := &visibilityPlaylistDS{ + MockDataStore: ds, + private: model.Playlist{ID: "plPrivate", OwnerID: "admin"}, + tracks: &tests.MockPlaylistTrackRepo{}, + } + + did, err := Backfill(ctx, vds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "plPrivate")).ToNot(BeNil()) + }) + It("enqueues artists before albums/playlists/radios, all at Backfill priority", func() { seedEntities() Expect(propRepo.Put(FingerprintPropertyKey, "stale-fingerprint")).To(Succeed()) diff --git a/core/artwork/worker.go b/core/artwork/worker.go index 4be32fee8..3b6ecc809 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -114,6 +114,9 @@ func (w *Worker) RunPrune(ctx context.Context) error { } func (w *Worker) drain(ctx context.Context, concurrency int) (int, error) { + // Resolved per drain, not once in Run: the worker starts at boot, possibly before any + // admin exists, so a late-created admin is picked up on the next poll (private playlists). + ctx = withAdminUser(ctx, w.deps.ds) batch, err := w.deps.ds.ArtworkQueue(ctx).DequeueBatch(2 * concurrency) if err != nil { return 0, err diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index 1fe904e21..8568d710e 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -213,6 +213,27 @@ var _ = Describe("Worker", func() { Expect(it.RetryAt).To(BeTemporally("==", dequeued.Add(time.Minute))) }) + It("resolves a private playlist under an admin context instead of failing forever", func() { + ds.MockedUser = adminUserRepo() + vds := &visibilityPlaylistDS{ + MockDataStore: ds, + private: model.Playlist{ID: "plPriv", OwnerID: "admin"}, + tracks: &tests.MockPlaylistTrackRepo{}, + } + w = NewWorker(vds, store, prov, ffm) + Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plPriv"})).To(Succeed()) + + n, err := w.drain(ctx, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(1)) + + // Resolved as absent (no art) and removed — not stuck failing on ErrNotFound forever. + Expect(findQueued(queueRepo, "pl", "plPriv")).To(BeNil()) + ia, err := artRepo.GetItemArtwork("pl", "plPriv", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).To(BeEmpty()) + }) + It("returns zero when the queue is empty", func() { n, err := w.drain(ctx, 2) Expect(err).ToNot(HaveOccurred()) From ba2290af6df469cf1d8d088617c03d422a69f2c4 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 15:28:54 -0400 Subject: [PATCH 46/51] test(artwork): convert non-synctest timing tests to Ginkgo specs TestArtworkBackoffSchedule and TestArtworkWorkerRunNoLeak needed no real *testing.T (no synctest), so move them into worker_test.go as Ginkgo specs. TestArtworkBreakerHalfOpen stays plain since testing/synctest requires a real *testing.T, matching core/scrobbler's precedent. --- core/artwork/worker_test.go | 54 +++++++++++++++++++++++++++ core/artwork/worker_timing_test.go | 60 +----------------------------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index 8568d710e..aebbb31bc 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -14,6 +14,7 @@ import ( "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/goleak" ) // reenqueueOnDequeue simulates a concurrent scan Enqueue between DequeueBatch and the @@ -304,5 +305,58 @@ var _ = Describe("Worker", func() { cancel() Eventually(done, time.Second).Should(Receive(BeNil())) }) + + It("does not leak goroutines after Run exits", func() { + DeferCleanup(configtest.SetupConfig()) + + ignore := goleak.IgnoreCurrent() + DeferCleanup(func() { goleak.VerifyNone(GinkgoT(), ignore) }) + + localDS := &tests.MockDataStore{MockedArtworkQueue: tests.CreateMockArtworkQueueRepo()} + lw := NewWorker(localDS, NewImageStore(GinkgoT().TempDir()), &fakeExternalProvider{}, tests.NewMockFFmpeg("")) + + runCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- lw.Run(runCtx) }() + + time.Sleep(20 * time.Millisecond) // let the loop settle on the idle select + cancel() + Eventually(done, 2*time.Second).Should(Receive(BeNil())) + }) + }) +}) + +var _ = Describe("backoff", func() { + It("returns the expected schedule with no jitter", func() { + for _, c := range []struct { + attempts int + want time.Duration + }{ + {0, 5 * time.Minute}, + {1, 20 * time.Minute}, + {2, 80 * time.Minute}, + {3, 320 * time.Minute}, + {4, 1280 * time.Minute}, + {5, 48 * time.Hour}, + {6, 48 * time.Hour}, + } { + Expect(backoffFor(c.attempts, 0)).To(Equal(c.want), "attempt %d", c.attempts) + } + }) + + It("applies jitter proportionally", func() { + base := backoffFor(2, 0) + Expect(backoffFor(2, 0.2)).To(Equal(time.Duration(float64(base) * 1.2))) + Expect(backoffFor(2, -0.2)).To(Equal(time.Duration(float64(base) * 0.8))) + }) + + It("keeps random jitter within +/-20%", func() { + lo := time.Duration(float64(320*time.Minute) * 0.8) + hi := time.Duration(float64(320*time.Minute) * 1.2) + for range 200 { + d := backoff(3) + Expect(d).To(BeNumerically(">=", lo)) + Expect(d).To(BeNumerically("<=", hi)) + } }) }) diff --git a/core/artwork/worker_timing_test.go b/core/artwork/worker_timing_test.go index a721459c2..4bb14df3a 100644 --- a/core/artwork/worker_timing_test.go +++ b/core/artwork/worker_timing_test.go @@ -1,48 +1,16 @@ package artwork import ( - "context" "errors" "testing" "testing/synctest" "time" - "github.com/navidrome/navidrome/conf/configtest" - "github.com/navidrome/navidrome/tests" . "github.com/onsi/gomega" - "go.uber.org/goleak" ) -func TestArtworkBackoffSchedule(t *testing.T) { - g := NewWithT(t) - for _, c := range []struct { - attempts int - want time.Duration - }{ - {0, 5 * time.Minute}, - {1, 20 * time.Minute}, - {2, 80 * time.Minute}, - {3, 320 * time.Minute}, - {4, 1280 * time.Minute}, - {5, 48 * time.Hour}, - {6, 48 * time.Hour}, - } { - g.Expect(backoffFor(c.attempts, 0)).To(Equal(c.want), "attempt %d", c.attempts) - } - - base := backoffFor(2, 0) - g.Expect(backoffFor(2, 0.2)).To(Equal(time.Duration(float64(base) * 1.2))) - g.Expect(backoffFor(2, -0.2)).To(Equal(time.Duration(float64(base) * 0.8))) - - lo := time.Duration(float64(320*time.Minute) * 0.8) - hi := time.Duration(float64(320*time.Minute) * 1.2) - for range 200 { - d := backoff(3) - g.Expect(d).To(BeNumerically(">=", lo)) - g.Expect(d).To(BeNumerically("<=", hi)) - } -} - +// Drives the real breaker state machine with the fake clock. Plain test: testing/synctest +// needs a *testing.T, which Ginkgo doesn't give. func TestArtworkBreakerHalfOpen(t *testing.T) { synctest.Test(t, func(t *testing.T) { g := NewWithT(t) @@ -69,27 +37,3 @@ func TestArtworkBreakerHalfOpen(t *testing.T) { g.Expect(b.allow()).To(BeTrue()) }) } - -func TestArtworkWorkerRunNoLeak(t *testing.T) { - ignore := goleak.IgnoreCurrent() - defer goleak.VerifyNone(t, ignore) - - t.Cleanup(configtest.SetupConfig()) - ds := &tests.MockDataStore{MockedArtworkQueue: tests.CreateMockArtworkQueueRepo()} - w := NewWorker(ds, NewImageStore(t.TempDir()), &fakeExternalProvider{}, tests.NewMockFFmpeg("")) - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- w.Run(ctx) }() - - time.Sleep(20 * time.Millisecond) // let the loop settle on the idle select - cancel() - select { - case err := <-done: - if err != nil { - t.Fatalf("Run returned error on cancel: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("Run did not exit after context cancel") - } -} From f614850ff0e6f8fe17d1758b24b80361e98c659e Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 15:45:01 -0400 Subject: [PATCH 47/51] fix(artwork): store backing-file provenance per item, not per hash --- core/artwork/processor.go | 22 ++++++------- core/artwork/processor_test.go | 60 +++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 58def5fc0..0158ca0c6 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -104,7 +104,8 @@ func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueI } art.SizeBytes = int64(len(data)) - if err := placeBytes(deps.store, art, res, data); err != nil { + sourcePath, refMtime, err := placeBytes(deps.store, art, res, data) + if err != nil { log.Warn(ctx, "artwork: failed to write image store", "kind", item.ItemKind, "id", item.ItemID, err) return outcomeFailed } @@ -118,6 +119,8 @@ func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueI ImageType: item.ImageType, Hash: hash, Source: res.source, + SourcePath: sourcePath, + RefMtime: refMtime, AttemptedAt: time.Now(), }); err != nil { log.Warn(ctx, "artwork: failed to persist item artwork state", "kind", item.ItemKind, "id", item.ItemID, err) @@ -196,21 +199,16 @@ func isFileBacked(source string) bool { return source == "folder" || source == "upload" } -// placeBytes fills in art's SourcePath/RefMtime and, for sources with no library file -// backing them, writes the bytes into the store (embedded keeps its audio file provenance). -func placeBytes(store *ImageStore, art *model.Artwork, res resolution, data []byte) error { +// placeBytes reports the item's backing-file provenance (folder/upload: image, embedded: audio, +// external/generated: none) and writes the bytes into the store for the non-file-backed sources. +func placeBytes(store *ImageStore, art *model.Artwork, res resolution, data []byte) (sourcePath string, refMtime int64, err error) { if isFileBacked(res.source) { - art.SourcePath = res.sourcePath - art.RefMtime = res.refMtime - return nil + return res.sourcePath, res.refMtime, nil } - art.SourcePath = "" - art.RefMtime = 0 if res.source == "embedded" { - art.SourcePath = res.sourcePath - art.RefMtime = res.refMtime + sourcePath, refMtime = res.sourcePath, res.refMtime } - return store.Write(art.Hash, art.Mime, bytes.NewReader(data)) + return sourcePath, refMtime, store.Write(art.Hash, art.Mime, bytes.NewReader(data)) } // mimeForFormat maps an image.Decode format name to its MIME type; extForMime diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go index 574bb106b..9ea4338fe 100644 --- a/core/artwork/processor_test.go +++ b/core/artwork/processor_test.go @@ -88,12 +88,11 @@ var _ = Describe("processItem", func() { Expect(err).ToNot(HaveOccurred()) Expect(ia.Hash).ToNot(BeEmpty()) Expect(ia.Source).To(Equal("folder")) + Expect(filepath.ToSlash(ia.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg")) + Expect(ia.RefMtime).To(BeNumerically(">", 0)) art, err := artRepo.GetImage(ia.Hash) Expect(err).ToNot(HaveOccurred()) - Expect(filepath.ToSlash(art.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg")) - Expect(art.RefMtime).To(BeNumerically(">", 0)) - _, err = store.Open(ia.Hash, art.Mime) Expect(os.IsNotExist(err)).To(BeTrue(), "folder-backed art must not be duplicated into the store") }) @@ -110,11 +109,11 @@ var _ = Describe("processItem", func() { ia, err := artRepo.GetItemArtwork("al", "al2", model.ImageTypePrimary) Expect(err).ToNot(HaveOccurred()) Expect(ia.Source).To(Equal("embedded")) + Expect(filepath.ToSlash(ia.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3")) art, err := artRepo.GetImage(ia.Hash) Expect(err).ToNot(HaveOccurred()) Expect(art.BlurHash).ToNot(BeEmpty()) - Expect(filepath.ToSlash(art.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3")) rc, err := store.Open(ia.Hash, art.Mime) Expect(err).ToNot(HaveOccurred()) @@ -207,6 +206,59 @@ var _ = Describe("processItem", func() { Expect(reused.BlurHash).To(Equal("SENTINEL")) }) + It("two items, two files, identical bytes: each item keeps its own provenance; the shared artwork row is written once", func() { + // Two distinct library files with byte-identical content resolve to the same + // hash. Provenance is per-item, so neither file's path may overwrite the other. + libRoot := GinkgoT().TempDir() + imgBytes, err := os.ReadFile(filepath.Join(repoRoot, "tests/fixtures/artist/an-album/cover.jpg")) + Expect(err).ToNot(HaveOccurred()) + for sub, mtime := range map[string]int64{"album-a": 1000, "album-b": 2000} { + dir := filepath.Join(libRoot, sub) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + img := filepath.Join(dir, "cover.jpg") + Expect(os.WriteFile(img, imgBytes, 0600)).To(Succeed()) + Expect(os.Chtimes(img, time.Unix(mtime, 0), time.Unix(mtime, 0))).To(Succeed()) + } + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ + {ID: "alA", Name: "Album A", FolderIDs: []string{"fa"}}, + {ID: "alB", Name: "Album B", FolderIDs: []string{"fb"}}, + }) + + folderRepo.result = []model.Folder{{Path: "album-a", ImageFiles: []string{"cover.jpg"}}} + Expect(processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alA"})).To(Equal(outcomeFound)) + iaA, err := artRepo.GetItemArtwork("al", "alA", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(iaA.Source).To(Equal("folder")) + Expect(filepath.ToSlash(iaA.SourcePath)).To(HaveSuffix("album-a/cover.jpg")) + Expect(iaA.RefMtime).To(Equal(int64(1000))) + + // Poison the shared row's blurhash: the second item must dedup on hash, not re-decode. + poisoned := artRepo.Data[iaA.Hash] + poisoned.BlurHash = "SENTINEL" + artRepo.Data[iaA.Hash] = poisoned + + folderRepo.result = []model.Folder{{Path: "album-b", ImageFiles: []string{"cover.jpg"}}} + Expect(processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alB"})).To(Equal(outcomeFound)) + iaB, err := artRepo.GetItemArtwork("al", "alB", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(iaB.Hash).To(Equal(iaA.Hash)) + Expect(filepath.ToSlash(iaB.SourcePath)).To(HaveSuffix("album-b/cover.jpg")) + Expect(iaB.RefMtime).To(Equal(int64(2000))) + + // The first item's provenance survives the second item processing identical bytes. + iaAafter, err := artRepo.GetItemArtwork("al", "alA", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(filepath.ToSlash(iaAafter.SourcePath)).To(HaveSuffix("album-a/cover.jpg")) + Expect(iaAafter.RefMtime).To(Equal(int64(1000))) + + // One shared artwork row, and dedup preserved it untouched. + Expect(artRepo.Data).To(HaveLen(1)) + reused, err := artRepo.GetImage(iaA.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(reused.BlurHash).To(Equal("SENTINEL")) + }) + It("decode failure on found bytes: fails without writing state", func() { tmpDir := GinkgoT().TempDir() conf.Server.DataFolder = conf.NewDir(tmpDir) From bba0eab3a556a3d2e2113b5836a464b0db27ba5c Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 17:09:22 -0400 Subject: [PATCH 48/51] fix(artwork): apply image limits to playlist tile decoding decodeTile ran image.Decode on every sampled album's resolved bytes before processItem's maxImageBytes/maxImagePixels guards applied, letting an oversized or decompression-bomb tile fully decode unbounded. Enforce both caps inside decodeTile itself. --- core/artwork/resolve.go | 18 +++++++++++++++++- core/artwork/resolve_test.go | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 8597ec836..984533309 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -389,8 +389,24 @@ func mtimeViaFS(fsys fs.FS, name string) int64 { // decodeTile and assembleTiles mirror playlistArtworkReader's createTile/ // createTiledImage, reusing the same rect/fillCenter cropping helpers. +// decodeTile runs on every sampled album's resolved bytes before processItem's +// own maxImageBytes/maxImagePixels guards apply, so it enforces them itself too. func decodeTile(r io.ReadCloser) (image.Image, error) { - img, _, err := image.Decode(r) + data, err := io.ReadAll(io.LimitReader(r, maxImageBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxImageBytes { + return nil, fmt.Errorf("tile image exceeds size cap %d", maxImageBytes) + } + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, err + } + if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels { + return nil, fmt.Errorf("tile dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels) + } + img, _, err := image.Decode(bytes.NewReader(data)) if err != nil { return nil, err } diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index 96702ef33..c64ee2b53 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -1,6 +1,7 @@ package artwork import ( + "bytes" "context" "errors" "image" @@ -509,6 +510,25 @@ var _ = Describe("resolveItem", func() { Expect(res.source).To(BeEmpty()) }) + It("skips a grid tile whose declared dimensions are a decompression bomb", func() { + // End-to-end regression: a bomb-declaring tile must not break the grid. + libRoot := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(libRoot, "bomb"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(libRoot, "bomb", "cover.jpg"), pngHeaderWithDims(50000, 50000), 0600)).To(Succeed()) + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + folderRepo.result = []model.Folder{{Path: "bomb", ImageFiles: []string{"cover.jpg"}}} + + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plbomb", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}} + ds.MockedPlaylist = plRepo + + res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plbomb"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(res.source).To(BeEmpty()) + }) + It("does not resolve as absent when every sampled album fails to resolve", func() { // "missing1"/"missing2" are not in MockAlbumRepo's data, so resolveAlbum // returns a genuine (non-external) error for every sampled tile. @@ -523,3 +543,20 @@ var _ = Describe("resolveItem", func() { }) }) }) + +// decodeTile runs on every sampled album's resolved bytes before processItem's +// own guards apply, so it must enforce the same caps independently. +var _ = Describe("decodeTile", func() { + It("rejects a decompression bomb before the full decode", func() { + data := pngHeaderWithDims(50000, 50000) // 2.5 gigapixels, far above the cap + _, err := decodeTile(io.NopCloser(bytes.NewReader(data))) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dimensions")) + }) + + It("rejects a tile larger than the size cap", func() { + data := bytes.Repeat([]byte{0}, maxImageBytes+1) + _, err := decodeTile(io.NopCloser(bytes.NewReader(data))) + Expect(err).To(HaveOccurred()) + }) +}) From b172ce429605aa5e589ca67565de268a261e7989 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 18:02:54 -0400 Subject: [PATCH 49/51] refactor(artwork): reuse auth.WithAdminUser and dedupe image cap guards --- core/artwork/housekeeping.go | 14 ++--------- core/artwork/processor.go | 46 +++++++++++++++++++++++++----------- core/artwork/resolve.go | 14 ++--------- core/artwork/worker.go | 3 ++- 4 files changed, 38 insertions(+), 39 deletions(-) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index 6d4cf4f43..fa7196ae6 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -9,9 +9,9 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/auth" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/utils/slice" ) @@ -35,20 +35,10 @@ func Fingerprint() string { return hex.EncodeToString(sum[:]) } -// withAdminUser wraps ctx with the first admin so repos that apply a per-user visibility -// filter (playlists) expose private rows during this headless work; no admin yet -> unchanged. -func withAdminUser(ctx context.Context, ds model.DataStore) context.Context { - admin, err := ds.User(ctx).FindFirstAdmin() - if err != nil || admin == nil || admin.ID == "" { - return ctx - } - return request.WithUser(ctx, *admin) -} - // Backfill enqueues artwork resolution for every entity when the config fingerprint changed // (or was never stored), artists first so those pages resolve before the larger backlog. func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { - ctx = withAdminUser(ctx, ds) + ctx = auth.WithAdminUser(ctx, ds) current := Fingerprint() props := ds.Property(ctx) stored, err := props.DefaultGet(FingerprintPropertyKey, "") diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 0158ca0c6..fefdebf84 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -71,13 +71,9 @@ func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueI } defer res.reader.Close() - data, err := io.ReadAll(io.LimitReader(res.reader, maxImageBytes+1)) + data, err := readCapped(res.reader) if err != nil { - log.Warn(ctx, "artwork: failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, err) - return outcomeFailed - } - if len(data) > maxImageBytes { - log.Warn(ctx, "artwork: resolved image exceeds size cap", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "cap", maxImageBytes) + log.Warn(ctx, "artwork: failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, err) return outcomeFailed } log.Debug(ctx, "artwork: read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "bytes", len(data)) @@ -147,19 +143,41 @@ func writeAbsent(ctx context.Context, repo model.ArtworkRepository, item model.A return outcomeAbsent } -// decodeArtwork builds a new Artwork row from raw bytes: dimensions, mime and a -// blurhash computed from a downscaled thumbnail. -func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwork, error) { +// readCapped reads r, rejecting anything over maxImageBytes. +func readCapped(r io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, maxImageBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxImageBytes { + return nil, fmt.Errorf("image exceeds size cap %d", maxImageBytes) + } + return data, nil +} + +// decodeCapped rejects declared dimensions over maxImagePixels BEFORE the +// full-decode allocation, then decodes. +func decodeCapped(data []byte) (image.Image, string, error) { cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) if err != nil { - return nil, fmt.Errorf("decode image config: %w", err) + return nil, "", fmt.Errorf("decode image config: %w", err) } if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels { - return nil, fmt.Errorf("image dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels) + return nil, "", fmt.Errorf("image dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels) } img, _, err := image.Decode(bytes.NewReader(data)) if err != nil { - return nil, fmt.Errorf("decode image: %w", err) + return nil, "", fmt.Errorf("decode image: %w", err) + } + return img, format, nil +} + +// decodeArtwork builds a new Artwork row from raw bytes: dimensions, mime and a +// blurhash computed from a downscaled thumbnail. +func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwork, error) { + img, format, err := decodeCapped(data) + if err != nil { + return nil, err } thumb := makeThumbnail(img, thumbnailSize) @@ -173,8 +191,8 @@ func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwor return &model.Artwork{ Hash: hash, Mime: mimeForFormat(format), - Width: cfg.Width, - Height: cfg.Height, + Width: img.Bounds().Dx(), + Height: img.Bounds().Dy(), BlurHash: bh, }, nil } diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 984533309..8a62b2999 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -392,21 +392,11 @@ func mtimeViaFS(fsys fs.FS, name string) int64 { // decodeTile runs on every sampled album's resolved bytes before processItem's // own maxImageBytes/maxImagePixels guards apply, so it enforces them itself too. func decodeTile(r io.ReadCloser) (image.Image, error) { - data, err := io.ReadAll(io.LimitReader(r, maxImageBytes+1)) + data, err := readCapped(r) if err != nil { return nil, err } - if len(data) > maxImageBytes { - return nil, fmt.Errorf("tile image exceeds size cap %d", maxImageBytes) - } - cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) - if err != nil { - return nil, err - } - if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels { - return nil, fmt.Errorf("tile dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels) - } - img, _, err := image.Decode(bytes.NewReader(data)) + img, _, err := decodeCapped(data) if err != nil { return nil, err } diff --git a/core/artwork/worker.go b/core/artwork/worker.go index 3b6ecc809..637af6efb 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -12,6 +12,7 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/core/external" "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/core/auth" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "golang.org/x/time/rate" @@ -116,7 +117,7 @@ func (w *Worker) RunPrune(ctx context.Context) error { func (w *Worker) drain(ctx context.Context, concurrency int) (int, error) { // Resolved per drain, not once in Run: the worker starts at boot, possibly before any // admin exists, so a late-created admin is picked up on the next poll (private playlists). - ctx = withAdminUser(ctx, w.deps.ds) + ctx = auth.WithAdminUser(ctx, w.deps.ds) batch, err := w.deps.ds.ArtworkQueue(ctx).DequeueBatch(2 * concurrency) if err != nil { return 0, err From 9ce51cf575f698a93393516144e7ae0415f32751 Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 23 Jul 2026 13:20:22 -0400 Subject: [PATCH 50/51] perf(artwork): fetch only IDs for backfill enumeration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backfill enumerated every album, artist, playlist and radio via GetAll and mapped out just the ID. GetAll materializes full entities (library joins, participant/stats/tags JSON, annotation, artwork hydration), so on a large library it loaded tens of thousands of heavy structs only to read one field each — spiking transient RSS to ~1GB during the one-time upgrade backfill, a memory risk on small NAS/Pi hardware. Add GetAllIDs to the album, artist, playlist and radio repositories: it reuses each repo's base row-set filter (library visibility, artist content join, playlist userFilter) but projects only id, skipping the heavy columns and post-processing. A per-repo parity test asserts GetAllIDs returns exactly the same id set as GetAll. Verified on a 727MB / 29k-artist production DB copy: peak RSS during backfill dropped from ~1012MB to ~89MB, file descriptors flat, same 36,138 items enqueued. --- core/artwork/housekeeping.go | 21 ++++----------------- model/album.go | 1 + model/artist.go | 1 + model/playlist.go | 1 + model/radio.go | 1 + persistence/album_repository.go | 9 +++++++++ persistence/album_repository_test.go | 15 +++++++++++++++ persistence/artist_repository.go | 9 +++++++++ persistence/artist_repository_test.go | 15 +++++++++++++++ persistence/playlist_repository.go | 10 ++++++++++ persistence/playlist_repository_test.go | 15 +++++++++++++++ persistence/radio_repository.go | 8 ++++++++ persistence/radio_repository_test.go | 15 +++++++++++++++ tests/mock_album_repo.go | 12 ++++++++++++ tests/mock_artist_repo.go | 12 ++++++++++++ tests/mock_playlist_repo.go | 12 ++++++++++++ tests/mock_radio_repository.go | 12 ++++++++++++ 17 files changed, 152 insertions(+), 17 deletions(-) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index fa7196ae6..f45c62ab6 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -12,7 +12,6 @@ import ( "github.com/navidrome/navidrome/core/auth" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/utils/slice" ) // FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against @@ -54,22 +53,10 @@ func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { kind string fetch func() ([]string, error) }{ - {"ar", func() ([]string, error) { - as, err := ds.Artist(ctx).GetAll() - return slice.Map(as, func(a model.Artist) string { return a.ID }), err - }}, - {"al", func() ([]string, error) { - as, err := ds.Album(ctx).GetAll() - return slice.Map(as, func(a model.Album) string { return a.ID }), err - }}, - {"pl", func() ([]string, error) { - ps, err := ds.Playlist(ctx).GetAll() - return slice.Map(ps, func(p model.Playlist) string { return p.ID }), err - }}, - {"ra", func() ([]string, error) { - rs, err := ds.Radio(ctx).GetAll() - return slice.Map(rs, func(r model.Radio) string { return r.ID }), err - }}, + {"ar", func() ([]string, error) { return ds.Artist(ctx).GetAllIDs() }}, + {"al", func() ([]string, error) { return ds.Album(ctx).GetAllIDs() }}, + {"pl", func() ([]string, error) { return ds.Playlist(ctx).GetAllIDs() }}, + {"ra", func() ([]string, error) { return ds.Radio(ctx).GetAllIDs() }}, } for _, k := range kinds { ids, err := k.fetch() diff --git a/model/album.go b/model/album.go index 55df8c63c..7c03ccf1e 100644 --- a/model/album.go +++ b/model/album.go @@ -142,6 +142,7 @@ type AlbumRepository interface { UpdateExternalInfo(*Album) error Get(id string) (*Album, error) GetAll(...QueryOptions) (Albums, error) + GetAllIDs(...QueryOptions) ([]string, error) GetCursor(...QueryOptions) (AlbumCursor, error) GetYears(libraryIDs ...int) ([]int, error) diff --git a/model/artist.go b/model/artist.go index f9c4bffd5..053f261ce 100644 --- a/model/artist.go +++ b/model/artist.go @@ -89,6 +89,7 @@ type ArtistRepository interface { UpdateExternalInfo(a *Artist) error Get(id string) (*Artist, error) GetAll(options ...QueryOptions) (Artists, error) + GetAllIDs(options ...QueryOptions) ([]string, error) GetCursor(options ...QueryOptions) (ArtistCursor, error) GetIndex(includeMissing bool, libraryIds []int, roles ...Role) (ArtistIndexes, error) diff --git a/model/playlist.go b/model/playlist.go index 185f6f942..b5726e4ff 100644 --- a/model/playlist.go +++ b/model/playlist.go @@ -143,6 +143,7 @@ type PlaylistRepository interface { Get(id string) (*Playlist, error) GetWithTracks(id string, refreshSmartPlaylist, includeMissing bool) (*Playlist, error) GetAll(options ...QueryOptions) (Playlists, error) + GetAllIDs(options ...QueryOptions) ([]string, error) GetCursor(options ...QueryOptions) (PlaylistCursor, error) FindByPath(path string) (*Playlist, error) Delete(id string) error diff --git a/model/radio.go b/model/radio.go index 86f27c24c..7f654b665 100644 --- a/model/radio.go +++ b/model/radio.go @@ -32,5 +32,6 @@ type RadioRepository interface { Delete(id string) error Get(id string) (*Radio, error) GetAll(options ...QueryOptions) (Radios, error) + GetAllIDs(options ...QueryOptions) ([]string, error) Put(u *Radio, colsToUpdate ...string) error } diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 2bb541003..e7b155e6f 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -254,6 +254,15 @@ func (r *albumRepository) GetAll(options ...model.QueryOptions) (model.Albums, e return res.toModels(), nil } +// GetAllIDs returns just the album IDs for the same row set as GetAll, skipping the +// heavy column projection and JSON post-processing. Used by bulk enumeration (artwork backfill). +func (r *albumRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sq := r.applyLibraryFilter(r.newSelect(options...).Columns("album.id")) + ids := []string{} + err := r.queryAllSlice(sq, &ids) + return ids, err +} + func (r *albumRepository) GetCursor(options ...model.QueryOptions) (model.AlbumCursor, error) { sq := r.selectAlbum(options...) cursor, err := queryWithStableResults[dbAlbum](r.sqlRepository, sq) diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 465617b10..ef18cf426 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -84,6 +84,21 @@ var _ = Describe("AlbumRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := albumRepo.GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, a := range want { + wantIDs = append(wantIDs, a.ID) + } + ids, err := albumRepo.GetAllIDs() + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("GetAll", func() { var GetAll = func(opts ...model.QueryOptions) (model.Albums, error) { albums, err := albumRepo.GetAll(opts...) diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index b542dedb4..f26ade8ba 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -264,6 +264,15 @@ func (r *artistRepository) GetAll(options ...model.QueryOptions) (model.Artists, return res, err } +// GetAllIDs returns just the artist IDs for the same row set as GetAll, skipping the +// heavy stats/annotation columns and JSON post-processing. Used by bulk enumeration (artwork backfill). +func (r *artistRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sq := r.applyLibraryFilterToArtistQuery(r.newSelect(options...).Columns("artist.id")).GroupBy("artist.id") + ids := []string{} + err := r.queryAllSlice(sq, &ids) + return ids, err +} + func (r *artistRepository) GetCursor(options ...model.QueryOptions) (model.ArtistCursor, error) { sel := r.selectArtist(options...) cursor, err := queryWithStableResults[dbArtist](r.sqlRepository, sel) diff --git a/persistence/artist_repository_test.go b/persistence/artist_repository_test.go index dc11ede36..43a7f4928 100644 --- a/persistence/artist_repository_test.go +++ b/persistence/artist_repository_test.go @@ -284,6 +284,21 @@ var _ = Describe("ArtistRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := repo.GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, a := range want { + wantIDs = append(wantIDs, a.ID) + } + ids, err := repo.GetAllIDs() + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("Basic Operations", func() { Describe("Count", func() { It("returns the number of artists in the DB", func() { diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.go index c78e0df1e..0abf53d93 100644 --- a/persistence/playlist_repository.go +++ b/persistence/playlist_repository.go @@ -189,6 +189,16 @@ func (r *playlistRepository) GetAll(options ...model.QueryOptions) (model.Playli return playlists, err } +// GetAllIDs returns just the playlist IDs for the same row set as GetAll (honoring userFilter), +// skipping the owner-name join columns and annotation. Used by bulk enumeration (artwork backfill). +func (r *playlistRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sq := r.newSelect(options...).Columns("playlist.id"). + Join("user on user.id = owner_id").Where(r.userFilter()) + ids := []string{} + err := r.queryAllSlice(sq, &ids) + return ids, err +} + func (r *playlistRepository) GetCursor(options ...model.QueryOptions) (model.PlaylistCursor, error) { // Same userFilter as GetAll: a cursor must not widen visibility beyond public/owned playlists. sel := r.selectPlaylist(options...).Where(r.userFilter()) diff --git a/persistence/playlist_repository_test.go b/persistence/playlist_repository_test.go index 00d4ff9f2..8dc86d887 100644 --- a/persistence/playlist_repository_test.go +++ b/persistence/playlist_repository_test.go @@ -37,6 +37,21 @@ var _ = Describe("PlaylistRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := repo.GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, p := range want { + wantIDs = append(wantIDs, p.ID) + } + ids, err := repo.GetAllIDs() + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("Exists", func() { It("returns true for an existing playlist", func() { Expect(repo.Exists(plsCool.ID)).To(BeTrue()) diff --git a/persistence/radio_repository.go b/persistence/radio_repository.go index 20f82ee36..3ef900920 100644 --- a/persistence/radio_repository.go +++ b/persistence/radio_repository.go @@ -59,6 +59,14 @@ func (r *radioRepository) GetAll(options ...model.QueryOptions) (model.Radios, e return res, err } +// GetAllIDs returns just the radio IDs. Used by bulk enumeration (artwork backfill). +func (r *radioRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + sel := r.newSelect(options...).Columns("id") + ids := []string{} + err := r.queryAllSlice(sel, &ids) + return ids, err +} + func (r *radioRepository) Put(radio *model.Radio, colsToUpdate ...string) error { if !r.isPermitted() { return rest.ErrPermissionDenied diff --git a/persistence/radio_repository_test.go b/persistence/radio_repository_test.go index 1d6cce4f8..4b243c142 100644 --- a/persistence/radio_repository_test.go +++ b/persistence/radio_repository_test.go @@ -78,6 +78,21 @@ var _ = Describe("RadioRepository", func() { }) }) + Describe("GetAllIDs", func() { + It("returns the same id set as GetAll", func() { + want, err := repo.GetAll() + Expect(err).To(BeNil()) + Expect(want).ToNot(BeEmpty()) + wantIDs := make([]string, 0, len(want)) + for _, r := range want { + wantIDs = append(wantIDs, r.ID) + } + ids, err := repo.GetAllIDs() + Expect(err).To(BeNil()) + Expect(ids).To(ConsistOf(wantIDs)) + }) + }) + Describe("Put", func() { It("successfully updates item", func() { err := repo.Put(&model.Radio{ diff --git a/tests/mock_album_repo.go b/tests/mock_album_repo.go index cc7d66b4d..17539255b 100644 --- a/tests/mock_album_repo.go +++ b/tests/mock_album_repo.go @@ -76,6 +76,18 @@ func (m *MockAlbumRepo) GetAll(qo ...model.QueryOptions) (model.Albums, error) { return m.All, nil } +func (m *MockAlbumRepo) GetAllIDs(qo ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(qo...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, a := range all { + ids[i] = a.ID + } + return ids, nil +} + func (m *MockAlbumRepo) GetCursor(qo ...model.QueryOptions) (model.AlbumCursor, error) { res, err := m.GetAll(qo...) if err != nil { diff --git a/tests/mock_artist_repo.go b/tests/mock_artist_repo.go index e6ea7aea4..4fdd198fe 100644 --- a/tests/mock_artist_repo.go +++ b/tests/mock_artist_repo.go @@ -113,6 +113,18 @@ func (m *MockArtistRepo) GetAll(options ...model.QueryOptions) (model.Artists, e return allArtists, nil } +func (m *MockArtistRepo) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(options...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, a := range all { + ids[i] = a.ID + } + return ids, nil +} + func (m *MockArtistRepo) GetCursor(options ...model.QueryOptions) (model.ArtistCursor, error) { res, err := m.GetAll(options...) if err != nil { diff --git a/tests/mock_playlist_repo.go b/tests/mock_playlist_repo.go index 8f8842c8e..bb1fe3379 100644 --- a/tests/mock_playlist_repo.go +++ b/tests/mock_playlist_repo.go @@ -52,6 +52,18 @@ func (m *MockPlaylistRepo) GetAll(options ...model.QueryOptions) (model.Playlist return m.All, nil } +func (m *MockPlaylistRepo) GetAllIDs(options ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(options...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, p := range all { + ids[i] = p.ID + } + return ids, nil +} + func (m *MockPlaylistRepo) GetCursor(options ...model.QueryOptions) (model.PlaylistCursor, error) { res, err := m.GetAll(options...) if err != nil { diff --git a/tests/mock_radio_repository.go b/tests/mock_radio_repository.go index c50a529e5..6d1d45f6e 100644 --- a/tests/mock_radio_repository.go +++ b/tests/mock_radio_repository.go @@ -73,6 +73,18 @@ func (m *MockedRadioRepo) GetAll(qo ...model.QueryOptions) (model.Radios, error) return m.All, nil } +func (m *MockedRadioRepo) GetAllIDs(qo ...model.QueryOptions) ([]string, error) { + all, err := m.GetAll(qo...) + if err != nil { + return nil, err + } + ids := make([]string, len(all)) + for i, r := range all { + ids[i] = r.ID + } + return ids, nil +} + func (m *MockedRadioRepo) Put(radio *model.Radio, _ ...string) error { if m.Err { return errors.New("error") From fc55e8bf16b517c124db89d2f70d77cf9477f99a Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 23 Jul 2026 14:05:10 -0400 Subject: [PATCH 51/51] feat(artwork): promote worker concurrency and external rate to real configs The artwork worker's drain speed was governed by two hidden Dev flags, DevArtworkWorkerConcurrency and DevArtworkExternalRPS, both defaulting to 2. On a large library's one-time backfill the external rate limiter is the real ceiling: every art-less item waits on it before the (rate-limited) external lookup, so the drain crawls at ~RPS items/sec while local-art items are unaffected. Promote both to documented, supported options: ArtworkWorkerConcurrency (default 4) sets local-resolution parallelism, ArtworkExternalMaxRPS (default 2, 0 = unlimited) caps external-agent lookups to stay polite to Last.fm/Deezer/etc. Operators can now trade first-backfill speed against external-API rate limits. The old Dev names still map for backward compat. --- conf/configuration.go | 10 ++++++---- core/artwork/worker.go | 6 +++--- core/artwork/worker_test.go | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/conf/configuration.go b/conf/configuration.go index 57ee4e4bc..9eeda89de 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -57,6 +57,8 @@ type configOptions struct { ImageCacheSize string AlbumPlayCountMode string EnableArtworkPrecache bool + ArtworkWorkerConcurrency int + ArtworkExternalMaxRPS int AutoImportPlaylists bool DefaultPlaylistPublicVisibility bool PlaylistsPath string @@ -139,8 +141,6 @@ type configOptions struct { DevArtworkThrottleBacklogLimit int DevArtworkThrottleBacklogTimeout time.Duration DevArtworkThrottleBuffered bool - DevArtworkWorkerConcurrency int - DevArtworkExternalRPS int DevArtistInfoTimeToLive time.Duration DevAlbumInfoTimeToLive time.Duration DevExternalScanner bool @@ -348,6 +348,8 @@ func Load(noConfigDump bool) { mapDeprecatedOption("CoverJpegQuality", "CoverArtQuality") mapDeprecatedOption("SimilarSongsMatchThreshold", "Matcher.FuzzyThreshold") mapDeprecatedOption("EnableTranscodingCancellation", "Transcoding.EnableCancellation") + mapDeprecatedOption("DevArtworkWorkerConcurrency", "ArtworkWorkerConcurrency") + mapDeprecatedOption("DevArtworkExternalRPS", "ArtworkExternalMaxRPS") err := viper.Unmarshal(&Server, viper.DecodeHook( mapstructure.ComposeDecodeHookFunc( @@ -902,8 +904,8 @@ func setViperDefaults() { viper.SetDefault("devartworkthrottlebackloglimit", consts.RequestThrottleBacklogLimit) viper.SetDefault("devartworkthrottlebacklogtimeout", consts.RequestThrottleBacklogTimeout) viper.SetDefault("devartworkthrottlebuffered", true) - viper.SetDefault("devartworkworkerconcurrency", 2) - viper.SetDefault("devartworkexternalrps", 2) + viper.SetDefault("artworkworkerconcurrency", 4) + viper.SetDefault("artworkexternalmaxrps", 2) viper.SetDefault("devartistinfotimetolive", consts.ArtistInfoTimeToLive) viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive) viper.SetDefault("devexternalscanner", true) diff --git a/core/artwork/worker.go b/core/artwork/worker.go index 637af6efb..5ae40a6e7 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -43,8 +43,8 @@ type Worker struct { } func NewWorker(ds model.DataStore, store *ImageStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg) *Worker { - rps := conf.Server.DevArtworkExternalRPS - limit := rate.Inf + rps := conf.Server.ArtworkExternalMaxRPS + limit := rate.Inf // 0 or negative disables the external throttle if rps > 0 { limit = rate.Limit(rps) } @@ -64,7 +64,7 @@ func NewWorker(ds model.DataStore, store *ImageStore, prov external.Provider, ff // leaked goroutines: each drain waits for its batch before the loop can return. func (w *Worker) Run(ctx context.Context) error { w.runCtx = ctx - concurrency := max(1, conf.Server.DevArtworkWorkerConcurrency) + concurrency := max(1, conf.Server.ArtworkWorkerConcurrency) ticker := time.NewTicker(workerPollInterval) defer ticker.Stop() for { diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index aebbb31bc..e1b0c985d 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -85,7 +85,7 @@ var _ = Describe("Worker", func() { ds.MockedAlbum = tests.CreateMockAlbumRepo() store = NewImageStore(GinkgoT().TempDir()) conf.Server.CoverArtPriority = "cover.jpg, embedded" - conf.Server.DevArtworkExternalRPS = 1000 // keep the limiter out of the way of behavior tests + conf.Server.ArtworkExternalMaxRPS = 1000 // keep the limiter out of the way of behavior tests w = NewWorker(ds, store, prov, ffm) })