diff --git a/core/artwork/image_store.go b/core/artwork/image_store.go new file mode 100644 index 000000000..4b4e482c9 --- /dev/null +++ b/core/artwork/image_store.go @@ -0,0 +1,164 @@ +package artwork + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/zeebo/xxh3" +) + +func HashImage(r io.Reader) (string, error) { + d := xxh3.New() + if _, err := io.Copy(d, r); err != nil { + return "", err + } + return fmt.Sprintf("%016x", d.Sum64()), nil +} + +// ImageStore is the content-addressed store for artwork images that have no +// library file backing them (external downloads, embedded extractions, generated). +type ImageStore struct { + root string +} + +func NewImageStore(rootDir string) *ImageStore { + return &ImageStore{root: rootDir} +} + +// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into +// content-addressed paths and re-derived on Open, so they must be stable across OSes. +func extForMime(m string) string { + switch m { + case "image/jpeg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + } + return ".img" +} + +// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "", +// and malformed persisted hashes must never reach path sharding (slice panics, separators). +func validHash(hash string) bool { + if len(hash) != 16 { + return false + } + for _, c := range []byte(hash) { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func (s *ImageStore) path(hash, mimeType string) string { + return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType)) +} + +func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error { + if !validHash(hash) { + return fmt.Errorf("imagestore: invalid hash %q", hash) + } + dst := s.path(hash, mimeType) + if _, err := os.Stat(dst); err == nil { + // A touched mtime marks the file live so a concurrent prune spares it. + now := time.Now() + if err := os.Chtimes(dst, now, now); err == nil { + return nil + } + // touch failed (file likely pruned concurrently) — fall through and write it + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, r); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dst) +} + +func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) { + if !validHash(hash) { + return nil, fmt.Errorf("imagestore: invalid hash %q", hash) + } + return os.Open(s.path(hash, mimeType)) +} + +// Remove deletes the store file unless it is newer than olderThan, in which case +// an overlapping acquisition may have just touched it and be about to commit its row. +func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error { + if !validHash(hash) { + return fmt.Errorf("imagestore: invalid hash %q", hash) + } + path := s.path(hash, mimeType) + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.ModTime().After(olderThan) { + return nil + } + err = os.Remove(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +// Sweep removes store files not accepted by keep. Files modified after cutoff +// (including temp files) are always kept: their acquisition row may not be committed yet. +func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) { + removed := 0 + err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().After(cutoff) { + return nil + } + name := d.Name() + remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window + if !remove { + ext := filepath.Ext(name) + remove = !keep(strings.TrimSuffix(name, ext), ext) + } + if remove { + // #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks + if err := os.Remove(path); err != nil { + return err + } + removed++ + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return removed, nil + } + return removed, err +} diff --git a/core/artwork/image_store_test.go b/core/artwork/image_store_test.go new file mode 100644 index 000000000..214cab3ad --- /dev/null +++ b/core/artwork/image_store_test.go @@ -0,0 +1,195 @@ +package artwork + +import ( + "bytes" + "io" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ImageStore", func() { + var store *ImageStore + var root string + + BeforeEach(func() { + root = GinkgoT().TempDir() + store = NewImageStore(root) + }) + + It("hashes deterministically", func() { + h1, err := HashImage(bytes.NewReader([]byte("some image bytes"))) + Expect(err).ToNot(HaveOccurred()) + h2, _ := HashImage(bytes.NewReader([]byte("some image bytes"))) + Expect(h1).To(Equal(h2)) + Expect(h1).To(HaveLen(16)) + h3, _ := HashImage(bytes.NewReader([]byte("other bytes"))) + Expect(h3).ToNot(Equal(h1)) + }) + + It("writes sharded and reads back", func() { + data := []byte("jpeg-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + got, _ := io.ReadAll(rc) + Expect(got).To(Equal(data)) + }) + + It("is idempotent on duplicate writes", func() { + data := []byte("dup") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + }) + + It("refreshes the mtime on a duplicate write", func() { + data := []byte("touch-me") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + + info, err := os.Stat(store.path(h, "image/png")) + Expect(err).ToNot(HaveOccurred()) + Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute))) + }) + + It("rewrites the bytes when the existing file vanished before the liveness touch", func() { + data := []byte("vanishing") + h, _ := HashImage(bytes.NewReader(data)) + for range 10 { + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(os.Remove(store.path(h, "image/png"))).To(Succeed()) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + + rc, err := store.Open(h, "image/png") + Expect(err).ToNot(HaveOccurred()) + got, _ := io.ReadAll(rc) + rc.Close() + Expect(got).To(Equal(data)) + } + }) + + It("returns fs.ErrNotExist for missing images", func() { + _, err := store.Open("beefbeefbeefbeef", "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("removes without error when already gone", func() { + Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed()) + }) + + It("rejects invalid hashes instead of panicking", func() { + for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} { + Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash"))) + _, err := store.Open(h, "image/jpeg") + Expect(err).To(MatchError(ContainSubstring("invalid hash"))) + Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash"))) + } + }) + + It("spares a file newer than the cutoff, removes an aged one", func() { + fresh := []byte("fresh") + hf, _ := HashImage(bytes.NewReader(fresh)) + Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed()) + + aged := []byte("aged") + ha, _ := HashImage(bytes.NewReader(aged)) + Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed()) + + cutoff := time.Now().Add(-time.Hour) + Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed()) + Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed()) + + rc, err := store.Open(hf, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + _, err = store.Open(ha, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("sweeps unknown files, keeps known ones", func() { + d1 := []byte("keep-me") + h1, _ := HashImage(bytes.NewReader(d1)) + Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed()) + d2 := []byte("orphan") + h2, _ := HashImage(bytes.NewReader(d2)) + Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed()) + + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h2, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h1, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("sweeps a stale mime variant of a known hash, keeps the current one", func() { + data := []byte("same-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + + // The recorded mime is image/jpeg, so the .png variant is obsolete. + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool { + return hash == h && ext == ".jpg" + }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h, "image/png") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("keeps young unknown files inside the grace window", func() { + d := []byte("fresh-orphan") + h, _ := HashImage(bytes.NewReader(d)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(0)) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("removes abandoned temp files past the grace window, keeps fresh ones", func() { + oldTmp := filepath.Join(root, ".old.tmp") + Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(oldTmp, old, old)).To(Succeed()) + + freshTmp := filepath.Join(root, ".fresh.tmp") + Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed()) + + removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + Expect(oldTmp).ToNot(BeAnExistingFile()) + Expect(freshTmp).To(BeAnExistingFile()) + }) +}) diff --git a/core/artwork/prune.go b/core/artwork/prune.go new file mode 100644 index 000000000..db1e4fdc1 --- /dev/null +++ b/core/artwork/prune.go @@ -0,0 +1,68 @@ +package artwork + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// pruneMinAge guards the window between artwork insert and item_artwork upsert. +const pruneMinAge = time.Hour + +func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error { + repo := ds.Artwork(ctx) + // One grace cutoff for both the DB orphan check and the file sweep: files younger + // than the window may belong to acquisitions whose rows aren't committed yet. + cutoff := time.Now().Add(-pruneMinAge) + candidates, err := repo.GetOrphanHashes(cutoff) + if err != nil { + return err + } + if len(candidates) > 0 { + arts, err := repo.GetImages(candidates) + if err != nil { + return err + } + if err := repo.DeleteOrphans(cutoff, candidates); err != nil { + return err + } + // DeleteOrphans may spare candidates reacquired since the snapshot; only remove files + // for rows actually gone (absent from the post-delete re-read). + survivors, err := repo.GetImages(candidates) + if err != nil { + return err + } + removed := 0 + for _, h := range candidates { + if _, ok := survivors[h]; ok { + continue + } + // A spared fresh file is at worst a stray a later sweep reclaims; full + // worker/prune mutual exclusion is Phase 2's concern. + if err := store.Remove(h, arts[h].Mime, cutoff); err != nil { + log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err) + } + removed++ + } + log.Info(ctx, "Prune: removed orphan artwork", "count", removed) + } + + mimes, err := repo.GetAllMimes() + if err != nil { + return err + } + removed, err := store.Sweep(cutoff, func(hash, ext string) bool { + // A known hash under a stale extension is a superseded mime variant — reclaim it. + m, ok := mimes[hash] + return ok && ext == extForMime(m) + }) + if err != nil { + return err + } + if removed > 0 { + log.Info(ctx, "Prune: swept stray artwork files", "count", removed) + } + return nil +} diff --git a/core/artwork/prune_test.go b/core/artwork/prune_test.go new file mode 100644 index 000000000..1f621363b --- /dev/null +++ b/core/artwork/prune_test.go @@ -0,0 +1,168 @@ +package artwork + +import ( + "bytes" + "context" + "errors" + "os" + "time" + + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type flakyGetArtworkRepo struct { + *tests.MockArtworkRepo +} + +func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) { + return nil, errors.New("db locked") +} + +var _ = Describe("Prune", func() { + var ds *tests.MockDataStore + var store *ImageStore + var awRepo *tests.MockArtworkRepo + + BeforeEach(func() { + ds = &tests.MockDataStore{} + awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo) + store = NewImageStore(GinkgoT().TempDir()) + }) + + // PutImage refreshes created_at like the SQL repo, so fixtures are aged directly. + ageArtwork := func(h string, t time.Time) { + a := awRepo.Data[h] + a.CreatedAt = t + awRepo.Data[h] = a + } + + It("deletes orphan rows and their store files, keeps referenced ones", func() { + data := []byte("orphan-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, old) + awRepo.OrphanHashes = []string{h} + + kept := []byte("kept-bytes") + hk, _ := HashImage(bytes.NewReader(kept)) + Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = store.Open(h, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(hk, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("spares a candidate reacquired between snapshot and delete", func() { + data := []byte("reacquired-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, time.Now().Add(-2*time.Hour)) + awRepo.OrphanHashes = []string{h} + // Reacquisition: an item now references the hash the snapshot flagged as orphan. + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", + ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).ToNot(HaveOccurred()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() { + data := []byte("fresh-reacquired-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + // Reacquisition refreshed created_at after the snapshot; still unreferenced. + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + awRepo.OrphanHashes = []string{h} + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := awRepo.GetImage(h) + Expect(err).ToNot(HaveOccurred()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("spares an orphan file freshly touched by an overlapping acquisition", func() { + data := []byte("racing-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + ageArtwork(h, time.Now().Add(-2*time.Hour)) + awRepo.OrphanHashes = []string{h} + // The row is legitimately orphaned, but a concurrent acquisition just touched the + // file's mtime (duplicate Write) and is about to commit a row referencing it. + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("sweeps store files that have no artwork row", func() { + stray := []byte("no-row-bytes") + h, _ := HashImage(bytes.NewReader(stray)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := store.Open(h, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("sweeps an obsolete mime variant of a reacquired hash", func() { + data := []byte("variant-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + old := time.Now().Add(-2 * time.Hour) + Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed()) + Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed()) + // The row records the current mime; the .png file is a superseded variant. + Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).To(Succeed()) + + _, err := store.Open(h, "image/png") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) + + It("never sweeps files on a transient DB error", func() { + ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()} + + data := []byte("live-bytes") + h, _ := HashImage(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(Prune(context.Background(), ds, store)).ToNot(Succeed()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) +}) diff --git a/db/migrations/20260722023032_add_artwork_tables.sql b/db/migrations/20260722023032_add_artwork_tables.sql new file mode 100644 index 000000000..bc400f90e --- /dev/null +++ b/db/migrations/20260722023032_add_artwork_tables.sql @@ -0,0 +1,42 @@ +-- +goose Up +CREATE TABLE artwork ( + hash TEXT PRIMARY KEY, + mime TEXT NOT NULL, + width INTEGER NOT NULL DEFAULT 0, + height INTEGER NOT NULL DEFAULT 0, + size_bytes INTEGER NOT NULL DEFAULT 0, + blur_hash TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE item_artwork ( + item_kind TEXT NOT NULL, + item_id TEXT NOT NULL, + image_type TEXT NOT NULL DEFAULT 'primary', + hash TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + source_path TEXT NOT NULL DEFAULT '', + ref_mtime INTEGER NOT NULL DEFAULT 0, + attempted_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY (item_kind, item_id, image_type) +) WITHOUT ROWID; +CREATE INDEX ix_item_artwork_hash ON item_artwork(hash); + +CREATE TABLE artwork_queue ( + item_kind TEXT NOT NULL, + item_id TEXT NOT NULL, + image_type TEXT NOT NULL DEFAULT 'primary', + priority INTEGER NOT NULL DEFAULT 0, + attempts INTEGER NOT NULL DEFAULT 0, + retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (item_kind, item_id, image_type) +) WITHOUT ROWID; +-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering. +CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at); + +-- +goose Down +DROP TABLE artwork_queue; +DROP TABLE item_artwork; +DROP TABLE artwork; diff --git a/go.mod b/go.mod index 5488b41e4..b72f4035c 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/tetratelabs/wazero v1.12.0 github.com/unrolled/secure v1.17.0 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 + github.com/zeebo/xxh3 v1.1.0 go.senan.xyz/taglib v0.11.1 go.uber.org/goleak v1.3.0 golang.org/x/image v0.44.0 @@ -128,7 +129,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect github.com/valyala/fastjson v1.6.10 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect diff --git a/model/artwork.go b/model/artwork.go new file mode 100644 index 000000000..7cb117764 --- /dev/null +++ b/model/artwork.go @@ -0,0 +1,93 @@ +package model + +import "time" + +// Artwork is one unique image, identified by the XXH3-64 hash of its bytes. +type Artwork struct { + Hash string `structs:"hash"` + Mime string `structs:"mime"` + Width int `structs:"width"` + Height int `structs:"height"` + SizeBytes int64 `structs:"size_bytes"` + BlurHash string `structs:"blur_hash"` + CreatedAt time.Time `structs:"created_at"` +} + +const ImageTypePrimary = "primary" + +// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent. +type ItemArtwork struct { + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Hash string `structs:"hash"` + Source string `structs:"source"` + // SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise. + SourcePath string `structs:"source_path"` + // RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath. + RefMtime int64 `structs:"ref_mtime"` + // attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork; + // raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL. + AttemptedAt time.Time `structs:"attempted_at"` + UpdatedAt time.Time `structs:"updated_at"` +} + +// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork). +type ItemArtworkInfo struct { + ItemID string + Hash string + BlurHash string +} + +// Absent reports a known-absent artwork state (resolved, no image). +func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" } + +type ArtworkQueueItem struct { + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Priority int `structs:"priority"` + Attempts int `structs:"attempts"` + RetryAt time.Time `structs:"retry_at"` + EnqueuedAt time.Time `structs:"enqueued_at"` +} + +// Queue priorities: higher drains first. +const ( + ArtworkPriorityRecheck = 0 + ArtworkPriorityBackfill = 10 + ArtworkPriorityScan = 50 + ArtworkPriorityBump = 100 +) + +type ArtworkRepository interface { + // Image identity (artwork table) + GetImage(hash string) (*Artwork, error) + PutImage(a *Artwork) error + GetImages(hashes []string) (map[string]Artwork, error) + // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. + GetOrphanHashes(createdBefore time.Time) ([]string, error) + // DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check). + DeleteOrphans(createdBefore time.Time, hashes []string) error + // Per-item state (item_artwork table) + GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error) + PutItemArtwork(ia *ItemArtwork) error + DeleteForItem(kind, id string) error + // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. + GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) + // GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks. + GetAllMimes() (map[string]string, error) +} + +type ArtworkQueueRepository interface { + // Enqueue upserts; an existing row keeps the higher of the two priorities. + Enqueue(items ...ArtworkQueueItem) error + // DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc. + DequeueBatch(n int) ([]ArtworkQueueItem, error) + // MarkFailed increments attempts and pushes retry_at into the future. + MarkFailed(kind, id, imageType string, retryAt time.Time) error + Delete(kind, id, imageType string) error + Count() (int64, error) + // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. + EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) +} diff --git a/model/datastore.go b/model/datastore.go index 94c3c3622..273ca714b 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -40,6 +40,8 @@ type DataStore interface { ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository Scrobble(ctx context.Context) ScrobbleRepository Plugin(ctx context.Context) PluginRepository + Artwork(ctx context.Context) ArtworkRepository + ArtworkQueue(ctx context.Context) ArtworkQueueRepository Resource(ctx context.Context, model any) ResourceRepository diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go new file mode 100644 index 000000000..ced5e9913 --- /dev/null +++ b/persistence/artwork_queue_repository.go @@ -0,0 +1,90 @@ +package persistence + +import ( + "context" + "slices" + "time" + + . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars). +const enqueueChunkSize = 100 + +type artworkQueueRepository struct { + sqlRepository +} + +func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository { + r := &artworkQueueRepository{} + r.ctx = ctx + r.db = db + r.tableName = "artwork_queue" + return r +} + +func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error { + now := time.Now() + for chunk := range slices.Chunk(items, enqueueChunkSize) { + ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at") + for _, it := range chunk { + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary + } + ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now) + } + ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`) + if _, err := r.executeSQL(ins); err != nil { + return err + } + } + return nil +} + +func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + sel := Select("*").From(r.tableName). + Where(LtOrEq{"retry_at": time.Now()}). + OrderBy("priority DESC", "enqueued_at ASC"). + Limit(uint64(n)) + var res []model.ArtworkQueueItem + err := r.queryAll(sel, &res) + return res, err +} + +func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error { + upd := Update(r.tableName). + Set("attempts", Expr("attempts + 1")). + Set("retry_at", retryAt). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + c, err := r.executeSQL(upd) + if err == nil && c == 0 { + return model.ErrNotFound + } + return err +} + +func (r *artworkQueueRepository) Delete(kind, id, imageType string) error { + return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) +} + +func (r *artworkQueueRepository) Count() (int64, error) { + var res struct{ Count int64 } + err := r.queryOne(Select("count(*) as count").From(r.tableName), &res) + return res.Count, err +} + +func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + now := time.Now() + // DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items. + ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at) + SELECT item_kind, item_id, image_type, ?, 0, ?, ? + FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ? + ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, + model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore) + return r.executeSQL(ins) +} + +var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil) diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go new file mode 100644 index 000000000..2b6f0d8c5 --- /dev/null +++ b/persistence/artwork_queue_repository_test.go @@ -0,0 +1,83 @@ +package persistence + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtworkQueueRepository", func() { + var repo model.ArtworkQueueRepository + + item := func(kind, id string, prio int) model.ArtworkQueueItem { + return model.ArtworkQueueItem{ItemKind: kind, ItemID: id, + ImageType: model.ImageTypePrimary, Priority: prio} + } + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder()) + }) + + It("enqueues and dequeues by priority then FIFO", func() { + Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed()) + Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed()) + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got[0].ItemID).To(Equal("high")) + }) + + It("keeps the higher priority on duplicate enqueue", func() { + Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed()) + Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed()) + got, _ := repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump)) + }) + + It("hides failed items until retry_at", func() { + Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed()) + Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed()) + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeEmpty()) + + Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed()) + got, _ = repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Attempts).To(Equal(2)) + }) + + It("deletes on completion and counts", func() { + Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed()) + n, _ := repo.Count() + Expect(n).To(Equal(int64(1))) + Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed()) + n, _ = repo.Count() + Expect(n).To(BeZero()) + }) + + It("enqueues stale absent states for recheck", func() { + awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) + old := time.Now().Add(-48 * time.Hour) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed()) + + n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + items, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(HaveLen(1)) + Expect(items[0].ItemID).To(Equal("stale1")) + Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck)) + }) +}) diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go new file mode 100644 index 000000000..505e3839d --- /dev/null +++ b/persistence/artwork_repository.go @@ -0,0 +1,179 @@ +package persistence + +import ( + "context" + "slices" + "time" + + . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" + "github.com/pocketbase/dbx" +) + +const ( + itemArtworkTable = "item_artwork" + artworkBatchSize = 200 +) + +type itemArtworkSQL struct { + sqlRepository +} + +type artworkRepository struct { + sqlRepository + items itemArtworkSQL +} + +func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository { + r := &artworkRepository{} + r.ctx = ctx + r.db = db + r.tableName = "artwork" + r.items.ctx = ctx + r.items.db = db + r.items.tableName = itemArtworkTable + return r +} + +func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) { + sel := Select("*").From(r.tableName).Where(Eq{"hash": hash}) + var res model.Artwork + if err := r.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil +} + +func (r *artworkRepository) PutImage(a *model.Artwork) error { + // created_at is the last-acquisition-write time the prune grace window keys on. + a.CreatedAt = time.Now() + values, err := toSQLArgs(*a) + if err != nil { + return err + } + // created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window. + ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, + height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`) + _, err = r.executeSQL(ins) + return err +} + +func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) { + res := map[string]model.Artwork{} + for chunk := range slices.Chunk(hashes, artworkBatchSize) { + sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk}) + var all []model.Artwork + if err := r.queryAll(sel, &all); err != nil { + return nil, err + } + for _, a := range all { + res[a.Hash] = a + } + } + return res, nil +} + +func (r *artworkRepository) GetAllMimes() (map[string]string, error) { + sel := Select("hash", "mime").From(r.tableName) + var rows []struct { + Hash string + Mime string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + res := make(map[string]string, len(rows)) + for _, row := range rows { + res[row.Hash] = row.Mime + } + return res, nil +} + +func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) { + sel := Select("hash").From(r.tableName). + Where(And{ + Lt{"created_at": createdBefore}, + Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"), + }) + var hashes []string + err := r.queryAllSlice(sel, &hashes) + return hashes, err +} + +func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error { + for chunk := range slices.Chunk(hashes, artworkBatchSize) { + del := Delete(r.tableName).Where(And{ + Eq{"hash": chunk}, + Lt{"created_at": createdBefore}, + Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"), + }) + if _, err := r.executeSQL(del); err != nil { + return err + } + } + return nil +} + +func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { + sel := Select("*").From(itemArtworkTable). + Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType}) + var res model.ItemArtwork + if err := r.items.queryOne(sel, &res); err != nil { + return nil, err + } + return &res, nil +} + +func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error { + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } + ia.UpdatedAt = time.Now() + // PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now. + if ia.AttemptedAt.IsZero() { + ia.AttemptedAt = ia.UpdatedAt + } + values, err := toSQLArgs(*ia) + if err != nil { + return err + } + ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET + hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime, + attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`) + _, err = r.items.executeSQL(ins) + return err +} + +func (r *artworkRepository) DeleteForItem(kind, id string) error { + return r.items.delete(Eq{"item_kind": kind, "item_id": id}) +} + +func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + res := map[string]model.ItemArtworkInfo{} + for chunk := range slices.Chunk(ids, artworkBatchSize) { + sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). + From(itemArtworkTable + " ia"). + LeftJoin("artwork a ON a.hash = ia.hash"). + Where(And{ + Eq{"ia.item_kind": kind}, + Eq{"ia.image_type": model.ImageTypePrimary}, + Eq{"ia.item_id": chunk}, + }) + var rows []struct { + ItemID string + Hash string + BlurHash string + } + if err := r.items.queryAll(sel, &rows); err != nil { + return nil, err + } + for _, row := range rows { + res[row.ItemID] = model.ItemArtworkInfo{ + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, + } + } + } + return res, nil +} + +var _ model.ArtworkRepository = (*artworkRepository)(nil) diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go new file mode 100644 index 000000000..82c2888c8 --- /dev/null +++ b/persistence/artwork_repository_test.go @@ -0,0 +1,197 @@ +package persistence + +import ( + "context" + "fmt" + "time" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pocketbase/dbx" +) + +// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state. +func clearArtworkTables() { + db := GetDBXBuilder() + for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} { + _, err := db.NewQuery("DELETE FROM " + t).Execute() + Expect(err).ToNot(HaveOccurred()) + } +} + +var _ = Describe("ArtworkRepository", func() { + var repo model.ArtworkRepository + + BeforeEach(func() { + clearArtworkTables() + repo = NewArtworkRepository(context.Background(), GetDBXBuilder()) + }) + + Context("image identity", func() { + It("stores and retrieves an artwork by hash", func() { + a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"} + Expect(repo.PutImage(a)).To(Succeed()) + + got, err := repo.GetImage("abc123") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Mime).To(Equal("image/jpeg")) + Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) + Expect(got.CreatedAt).ToNot(BeZero()) + }) + + It("is idempotent on Put (upsert by hash)", func() { + a := &model.Artwork{Hash: "dup1", Mime: "image/png"} + Expect(repo.PutImage(a)).To(Succeed()) + a.BlurHash = "XYZ" + Expect(repo.PutImage(a)).To(Succeed()) + got, _ := repo.GetImage("dup1") + Expect(got.BlurHash).To(Equal("XYZ")) + }) + + It("refreshes created_at when reacquiring an existing hash", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed()) + _, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'"). + Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute() + Expect(err).ToNot(HaveOccurred()) + + Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed()) + + got, err := repo.GetImage("reacq") + Expect(err).ToNot(HaveOccurred()) + Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) + }) + + It("returns ErrNotFound for a missing hash", func() { + _, err := repo.GetImage("nope") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + It("fetches a batch", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed()) + got, err := repo.GetImages([]string{"b1", "b2", "missing"}) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(2)) + Expect(got["b2"].Mime).To(Equal("image/png")) + }) + + It("returns every stored hash with its current mime", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed()) + mimes, err := repo.GetAllMimes() + Expect(err).ToNot(HaveOccurred()) + Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg")) + Expect(mimes).To(HaveKeyWithValue("all2", "image/png")) + }) + + It("finds orphans older than cutoff, honoring item_artwork references", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed()) + + orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(ContainElement("orph1")) + Expect(orphans).ToNot(ContainElement("ref1")) + + orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour)) + Expect(err).ToNot(HaveOccurred()) + Expect(orphans).To(BeEmpty()) + }) + + It("deletes only unreferenced hashes older than the cutoff", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", + ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed()) + + Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed()) + + _, err := repo.GetImage("d1") + Expect(err).To(MatchError(model.ErrNotFound)) + _, err = repo.GetImage("dref") + Expect(err).ToNot(HaveOccurred()) + }) + + It("spares an unreferenced hash younger than the cutoff", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed()) + Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed()) + _, err := repo.GetImage("young") + Expect(err).ToNot(HaveOccurred()) + }) + + It("fetches a batch larger than the SQL variable limit", func() { + hashes := make([]string, 0, 250) + for i := range 250 { + h := fmt.Sprintf("big%03d", i) + Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + hashes = append(hashes, h) + } + hashes = append(hashes, "absent1", "absent2") + + got, err := repo.GetImages(hashes) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(250)) + }) + }) + + Context("item state", func() { + It("upserts and reads state, including per-item provenance", func() { + ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, + Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()} + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + ia.Source = "embedded" + ia.SourcePath = "/music/a/track.mp3" + ia.RefMtime = 222 + Expect(repo.PutItemArtwork(ia)).To(Succeed()) + + got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Source).To(Equal("embedded")) + Expect(got.SourcePath).To(Equal("/music/a/track.mp3")) + Expect(got.RefMtime).To(Equal(int64(222))) + Expect(got.UpdatedAt).ToNot(BeZero()) + }) + + It("defaults attempted_at to now when unset", func() { + before := time.Now().Add(-time.Second) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt", + ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.AttemptedAt).To(BeTemporally(">", before)) + }) + + It("represents known-absent as empty hash", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", + ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed()) + got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Hash).To(BeEmpty()) + }) + + It("hydrates a page in one batch, including blurhash and absence", func() { + Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) + + info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(info).To(HaveLen(2)) + Expect(info["x1"].Hash).To(Equal("h9")) + Expect(info["x1"].BlurHash).To(Equal("BH9")) + Expect(info["x1"].Absent()).To(BeFalse()) + Expect(info["x2"].Absent()).To(BeTrue()) + _, unresolved := info["x3"] + Expect(unresolved).To(BeFalse()) + }) + + It("deletes all rows for an item", func() { + Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed()) + Expect(repo.DeleteForItem("pl", "p1")).To(Succeed()) + _, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + }) +}) diff --git a/persistence/persistence.go b/persistence/persistence.go index 93f0e3e71..9d3a33cfc 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -97,6 +97,14 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository { return NewPluginRepository(ctx, s.getDBXBuilder()) } +func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository { + return NewArtworkRepository(ctx, s.getDBXBuilder()) +} + +func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { + return NewArtworkQueueRepository(ctx, s.getDBXBuilder()) +} + func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository { switch m.(type) { case model.User: diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go new file mode 100644 index 000000000..f62afe43a --- /dev/null +++ b/tests/mock_artwork_queue_repo.go @@ -0,0 +1,125 @@ +package tests + +import ( + "sort" + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockArtworkQueueRepo struct { + model.ArtworkQueueRepository + Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType) + Err error + // ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state. + ItemArtworkSource *MockArtworkRepo +} + +func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo { + return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}} +} + +func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { + if m.Err != nil { + return m.Err + } + now := time.Now() + for _, it := range items { + if it.ImageType == "" { + it.ImageType = model.ImageTypePrimary + } + k := iaKey(it.ItemKind, it.ItemID, it.ImageType) + // Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller. + if prev, ok := m.Data[k]; ok { + prev.Priority = max(prev.Priority, it.Priority) + prev.RetryAt = now + m.Data[k] = prev + continue + } + it.Attempts = 0 + it.RetryAt = now + it.EnqueuedAt = now + m.Data[k] = it + } + return nil +} + +func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) { + if m.Err != nil { + return nil, m.Err + } + var res []model.ArtworkQueueItem + now := time.Now() + for _, it := range m.Data { + if !it.RetryAt.After(now) { + res = append(res, it) + } + } + sort.Slice(res, func(i, j int) bool { + if res[i].Priority != res[j].Priority { + return res[i].Priority > res[j].Priority + } + return res[i].EnqueuedAt.Before(res[j].EnqueuedAt) + }) + if len(res) > n { + res = res[:n] + } + return res, nil +} + +func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error { + if m.Err != nil { + return m.Err + } + k := iaKey(kind, id, imageType) + it, ok := m.Data[k] + if !ok { + return model.ErrNotFound + } + it.Attempts++ + it.RetryAt = retryAt + m.Data[k] = it + return nil +} + +func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error { + if m.Err != nil { + return m.Err + } + delete(m.Data, iaKey(kind, id, imageType)) + return nil +} + +func (m *MockArtworkQueueRepo) Count() (int64, error) { + if m.Err != nil { + return 0, m.Err + } + return int64(len(m.Data)), nil +} + +func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) { + if m.Err != nil || m.ItemArtworkSource == nil { + return 0, m.Err + } + now := time.Now() + var inserted int64 + for _, ia := range m.ItemArtworkSource.ItemData { + if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) { + continue + } + k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType) + if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows + continue + } + m.Data[k] = model.ArtworkQueueItem{ + ItemKind: ia.ItemKind, + ItemID: ia.ItemID, + ImageType: ia.ImageType, + Priority: model.ArtworkPriorityRecheck, + RetryAt: now, + EnqueuedAt: now, + } + inserted++ + } + return inserted, nil +} diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go new file mode 100644 index 000000000..13ad4d5e1 --- /dev/null +++ b/tests/mock_artwork_repo.go @@ -0,0 +1,149 @@ +package tests + +import ( + "time" + + "github.com/navidrome/navidrome/model" +) + +type MockArtworkRepo struct { + model.ArtworkRepository + Data map[string]model.Artwork + ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType) + OrphanHashes []string + Err error +} + +func CreateMockArtworkRepo() *MockArtworkRepo { + return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}} +} + +func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType } + +func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) { + if m.Err != nil { + return nil, m.Err + } + if a, ok := m.Data[hash]; ok { + return &a, nil + } + return nil, model.ErrNotFound +} + +func (m *MockArtworkRepo) PutImage(a *model.Artwork) error { + if m.Err != nil { + return m.Err + } + // Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly. + a.CreatedAt = time.Now() + m.Data[a.Hash] = *a + return nil +} + +func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.Artwork{} + for _, h := range hashes { + if a, ok := m.Data[h]; ok { + res[h] = a + } + } + return res, nil +} + +func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) { + return m.OrphanHashes, m.Err +} + +func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) { + if m.Err != nil { + return nil, m.Err + } + mimes := make(map[string]string, len(m.Data)) + for h, a := range m.Data { + mimes[h] = a.Mime + } + return mimes, nil +} + +func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error { + if m.Err != nil { + return m.Err + } + // Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted. + for _, h := range hashes { + if m.referenced(h) { + continue + } + if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) { + continue + } + delete(m.Data, h) + } + return nil +} + +func (m *MockArtworkRepo) referenced(hash string) bool { + for _, ia := range m.ItemData { + if ia.Hash == hash { + return true + } + } + return false +} + +func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { + if m.Err != nil { + return nil, m.Err + } + if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok { + return &ia, nil + } + return nil, model.ErrNotFound +} + +func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error { + if m.Err != nil { + return m.Err + } + if ia.ImageType == "" { + ia.ImageType = model.ImageTypePrimary + } + ia.UpdatedAt = time.Now() + if ia.AttemptedAt.IsZero() { + ia.AttemptedAt = ia.UpdatedAt + } + m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia + return nil +} + +func (m *MockArtworkRepo) DeleteForItem(kind, id string) error { + if m.Err != nil { + return m.Err + } + for k, ia := range m.ItemData { + if ia.ItemKind == kind && ia.ItemID == id { + delete(m.ItemData, k) + } + } + return nil +} + +func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { + if m.Err != nil { + return nil, m.Err + } + res := map[string]model.ItemArtworkInfo{} + for _, id := range ids { + if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok { + info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash} + if a, ok := m.Data[ia.Hash]; ok { + info.BlurHash = a.BlurHash + } + res[id] = info + } + } + return res, nil +} diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index e016a28de..d8585fabf 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -28,6 +28,8 @@ type MockDataStore struct { MockedScrobble model.ScrobbleRepository MockedRadio model.RadioRepository MockedPlugin model.PluginRepository + MockedArtwork model.ArtworkRepository + MockedArtworkQueue model.ArtworkQueueRepository scrobbleBufferMu sync.Mutex repoMu sync.Mutex @@ -247,6 +249,32 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository { return db.MockedPlugin } +func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository { + if db.MockedArtwork != nil { + return db.MockedArtwork + } + if db.RealDS != nil { + return db.RealDS.Artwork(ctx) + } + db.MockedArtwork = CreateMockArtworkRepo() + return db.MockedArtwork +} + +func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository { + if db.MockedArtworkQueue != nil { + return db.MockedArtworkQueue + } + if db.RealDS != nil { + return db.RealDS.ArtworkQueue(ctx) + } + q := CreateMockArtworkQueueRepo() + if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok { + q.ItemArtworkSource = aw + } + db.MockedArtworkQueue = q + return db.MockedArtworkQueue +} + func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error { return block(db) }