mirror of
https://github.com/navidrome/navidrome.git
synced 2026-10-14 05:17:13 +02:00
Compare commits
21 commits
master
...
artwork-fo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f818e7633 | ||
|
|
c04c8ee02a | ||
|
|
ebbe533c6a | ||
|
|
0147cc59b1 | ||
|
|
034cd17498 | ||
|
|
8147f7c40b | ||
|
|
bf614e66ad | ||
|
|
623b7d6a6c | ||
|
|
6f7f9c6463 | ||
|
|
8fd7ef19f3 | ||
|
|
1041e45ca7 | ||
|
|
4f835437a9 | ||
|
|
b16ef725c9 | ||
|
|
3e7685adc2 | ||
|
|
db16b3de9a | ||
|
|
b72597821a | ||
|
|
fcff9c63e7 | ||
|
|
14dd57052e | ||
|
|
f926539c04 | ||
|
|
6cce65f759 | ||
|
|
7aacb01f4f |
16 changed files with 1592 additions and 1 deletions
164
core/artwork/image_store.go
Normal file
164
core/artwork/image_store.go
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
package artwork
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/zeebo/xxh3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func HashImage(r io.Reader) (string, error) {
|
||||||
|
d := xxh3.New()
|
||||||
|
if _, err := io.Copy(d, r); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%016x", d.Sum64()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImageStore is the content-addressed store for artwork images that have no
|
||||||
|
// library file backing them (external downloads, embedded extractions, generated).
|
||||||
|
type ImageStore struct {
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewImageStore(rootDir string) *ImageStore {
|
||||||
|
return &ImageStore{root: rootDir}
|
||||||
|
}
|
||||||
|
|
||||||
|
// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into
|
||||||
|
// content-addressed paths and re-derived on Open, so they must be stable across OSes.
|
||||||
|
func extForMime(m string) string {
|
||||||
|
switch m {
|
||||||
|
case "image/jpeg":
|
||||||
|
return ".jpg"
|
||||||
|
case "image/png":
|
||||||
|
return ".png"
|
||||||
|
case "image/gif":
|
||||||
|
return ".gif"
|
||||||
|
case "image/webp":
|
||||||
|
return ".webp"
|
||||||
|
}
|
||||||
|
return ".img"
|
||||||
|
}
|
||||||
|
|
||||||
|
// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "",
|
||||||
|
// and malformed persisted hashes must never reach path sharding (slice panics, separators).
|
||||||
|
func validHash(hash string) bool {
|
||||||
|
if len(hash) != 16 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, c := range []byte(hash) {
|
||||||
|
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ImageStore) path(hash, mimeType string) string {
|
||||||
|
return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error {
|
||||||
|
if !validHash(hash) {
|
||||||
|
return fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||||
|
}
|
||||||
|
dst := s.path(hash, mimeType)
|
||||||
|
if _, err := os.Stat(dst); err == nil {
|
||||||
|
// A touched mtime marks the file live so a concurrent prune spares it.
|
||||||
|
now := time.Now()
|
||||||
|
if err := os.Chtimes(dst, now, now); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// touch failed (file likely pruned concurrently) — fall through and write it
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmp.Name())
|
||||||
|
if _, err := io.Copy(tmp, r); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp.Name(), dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) {
|
||||||
|
if !validHash(hash) {
|
||||||
|
return nil, fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||||
|
}
|
||||||
|
return os.Open(s.path(hash, mimeType))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove deletes the store file unless it is newer than olderThan, in which case
|
||||||
|
// an overlapping acquisition may have just touched it and be about to commit its row.
|
||||||
|
func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error {
|
||||||
|
if !validHash(hash) {
|
||||||
|
return fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||||
|
}
|
||||||
|
path := s.path(hash, mimeType)
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.ModTime().After(olderThan) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err = os.Remove(path)
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sweep removes store files not accepted by keep. Files modified after cutoff
|
||||||
|
// (including temp files) are always kept: their acquisition row may not be committed yet.
|
||||||
|
func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) {
|
||||||
|
removed := 0
|
||||||
|
err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := d.Info()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.ModTime().After(cutoff) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
name := d.Name()
|
||||||
|
remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window
|
||||||
|
if !remove {
|
||||||
|
ext := filepath.Ext(name)
|
||||||
|
remove = !keep(strings.TrimSuffix(name, ext), ext)
|
||||||
|
}
|
||||||
|
if remove {
|
||||||
|
// #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return removed, nil
|
||||||
|
}
|
||||||
|
return removed, err
|
||||||
|
}
|
||||||
195
core/artwork/image_store_test.go
Normal file
195
core/artwork/image_store_test.go
Normal file
|
|
@ -0,0 +1,195 @@
|
||||||
|
package artwork
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
. "github.com/onsi/ginkgo/v2"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ = Describe("ImageStore", func() {
|
||||||
|
var store *ImageStore
|
||||||
|
var root string
|
||||||
|
|
||||||
|
BeforeEach(func() {
|
||||||
|
root = GinkgoT().TempDir()
|
||||||
|
store = NewImageStore(root)
|
||||||
|
})
|
||||||
|
|
||||||
|
It("hashes deterministically", func() {
|
||||||
|
h1, err := HashImage(bytes.NewReader([]byte("some image bytes")))
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
h2, _ := HashImage(bytes.NewReader([]byte("some image bytes")))
|
||||||
|
Expect(h1).To(Equal(h2))
|
||||||
|
Expect(h1).To(HaveLen(16))
|
||||||
|
h3, _ := HashImage(bytes.NewReader([]byte("other bytes")))
|
||||||
|
Expect(h3).ToNot(Equal(h1))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("writes sharded and reads back", func() {
|
||||||
|
data := []byte("jpeg-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
|
||||||
|
Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile())
|
||||||
|
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
defer rc.Close()
|
||||||
|
got, _ := io.ReadAll(rc)
|
||||||
|
Expect(got).To(Equal(data))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("is idempotent on duplicate writes", func() {
|
||||||
|
data := []byte("dup")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("refreshes the mtime on a duplicate write", func() {
|
||||||
|
data := []byte("touch-me")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||||
|
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
|
||||||
|
info, err := os.Stat(store.path(h, "image/png"))
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute)))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("rewrites the bytes when the existing file vanished before the liveness touch", func() {
|
||||||
|
data := []byte("vanishing")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
for range 10 {
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
Expect(os.Remove(store.path(h, "image/png"))).To(Succeed())
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
|
||||||
|
rc, err := store.Open(h, "image/png")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
got, _ := io.ReadAll(rc)
|
||||||
|
rc.Close()
|
||||||
|
Expect(got).To(Equal(data))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
It("returns fs.ErrNotExist for missing images", func() {
|
||||||
|
_, err := store.Open("beefbeefbeefbeef", "image/jpeg")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("removes without error when already gone", func() {
|
||||||
|
Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("rejects invalid hashes instead of panicking", func() {
|
||||||
|
for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} {
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash")))
|
||||||
|
_, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).To(MatchError(ContainSubstring("invalid hash")))
|
||||||
|
Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash")))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
It("spares a file newer than the cutoff, removes an aged one", func() {
|
||||||
|
fresh := []byte("fresh")
|
||||||
|
hf, _ := HashImage(bytes.NewReader(fresh))
|
||||||
|
Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed())
|
||||||
|
|
||||||
|
aged := []byte("aged")
|
||||||
|
ha, _ := HashImage(bytes.NewReader(aged))
|
||||||
|
Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed())
|
||||||
|
|
||||||
|
cutoff := time.Now().Add(-time.Hour)
|
||||||
|
Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed())
|
||||||
|
Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed())
|
||||||
|
|
||||||
|
rc, err := store.Open(hf, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
_, err = store.Open(ha, "image/jpeg")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("sweeps unknown files, keeps known ones", func() {
|
||||||
|
d1 := []byte("keep-me")
|
||||||
|
h1, _ := HashImage(bytes.NewReader(d1))
|
||||||
|
Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed())
|
||||||
|
d2 := []byte("orphan")
|
||||||
|
h2, _ := HashImage(bytes.NewReader(d2))
|
||||||
|
Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed())
|
||||||
|
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed())
|
||||||
|
|
||||||
|
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 })
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(removed).To(Equal(1))
|
||||||
|
_, err = store.Open(h2, "image/jpeg")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
rc, err := store.Open(h1, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("sweeps a stale mime variant of a known hash, keeps the current one", func() {
|
||||||
|
data := []byte("same-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||||
|
|
||||||
|
// The recorded mime is image/jpeg, so the .png variant is obsolete.
|
||||||
|
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool {
|
||||||
|
return hash == h && ext == ".jpg"
|
||||||
|
})
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(removed).To(Equal(1))
|
||||||
|
_, err = store.Open(h, "image/png")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("keeps young unknown files inside the grace window", func() {
|
||||||
|
d := []byte("fresh-orphan")
|
||||||
|
h, _ := HashImage(bytes.NewReader(d))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed())
|
||||||
|
|
||||||
|
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false })
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(removed).To(Equal(0))
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("removes abandoned temp files past the grace window, keeps fresh ones", func() {
|
||||||
|
oldTmp := filepath.Join(root, ".old.tmp")
|
||||||
|
Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(oldTmp, old, old)).To(Succeed())
|
||||||
|
|
||||||
|
freshTmp := filepath.Join(root, ".fresh.tmp")
|
||||||
|
Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed())
|
||||||
|
|
||||||
|
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true })
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(removed).To(Equal(1))
|
||||||
|
Expect(oldTmp).ToNot(BeAnExistingFile())
|
||||||
|
Expect(freshTmp).To(BeAnExistingFile())
|
||||||
|
})
|
||||||
|
})
|
||||||
68
core/artwork/prune.go
Normal file
68
core/artwork/prune.go
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
package artwork
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/navidrome/navidrome/log"
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pruneMinAge guards the window between artwork insert and item_artwork upsert.
|
||||||
|
const pruneMinAge = time.Hour
|
||||||
|
|
||||||
|
func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error {
|
||||||
|
repo := ds.Artwork(ctx)
|
||||||
|
// One grace cutoff for both the DB orphan check and the file sweep: files younger
|
||||||
|
// than the window may belong to acquisitions whose rows aren't committed yet.
|
||||||
|
cutoff := time.Now().Add(-pruneMinAge)
|
||||||
|
candidates, err := repo.GetOrphanHashes(cutoff)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(candidates) > 0 {
|
||||||
|
arts, err := repo.GetImages(candidates)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := repo.DeleteOrphans(cutoff, candidates); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// DeleteOrphans may spare candidates reacquired since the snapshot; only remove files
|
||||||
|
// for rows actually gone (absent from the post-delete re-read).
|
||||||
|
survivors, err := repo.GetImages(candidates)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
removed := 0
|
||||||
|
for _, h := range candidates {
|
||||||
|
if _, ok := survivors[h]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A spared fresh file is at worst a stray a later sweep reclaims; full
|
||||||
|
// worker/prune mutual exclusion is Phase 2's concern.
|
||||||
|
if err := store.Remove(h, arts[h].Mime, cutoff); err != nil {
|
||||||
|
log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err)
|
||||||
|
}
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
log.Info(ctx, "Prune: removed orphan artwork", "count", removed)
|
||||||
|
}
|
||||||
|
|
||||||
|
mimes, err := repo.GetAllMimes()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
removed, err := store.Sweep(cutoff, func(hash, ext string) bool {
|
||||||
|
// A known hash under a stale extension is a superseded mime variant — reclaim it.
|
||||||
|
m, ok := mimes[hash]
|
||||||
|
return ok && ext == extForMime(m)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if removed > 0 {
|
||||||
|
log.Info(ctx, "Prune: swept stray artwork files", "count", removed)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
168
core/artwork/prune_test.go
Normal file
168
core/artwork/prune_test.go
Normal file
|
|
@ -0,0 +1,168 @@
|
||||||
|
package artwork
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
"github.com/navidrome/navidrome/tests"
|
||||||
|
. "github.com/onsi/ginkgo/v2"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
)
|
||||||
|
|
||||||
|
type flakyGetArtworkRepo struct {
|
||||||
|
*tests.MockArtworkRepo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) {
|
||||||
|
return nil, errors.New("db locked")
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ = Describe("Prune", func() {
|
||||||
|
var ds *tests.MockDataStore
|
||||||
|
var store *ImageStore
|
||||||
|
var awRepo *tests.MockArtworkRepo
|
||||||
|
|
||||||
|
BeforeEach(func() {
|
||||||
|
ds = &tests.MockDataStore{}
|
||||||
|
awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo)
|
||||||
|
store = NewImageStore(GinkgoT().TempDir())
|
||||||
|
})
|
||||||
|
|
||||||
|
// PutImage refreshes created_at like the SQL repo, so fixtures are aged directly.
|
||||||
|
ageArtwork := func(h string, t time.Time) {
|
||||||
|
a := awRepo.Data[h]
|
||||||
|
a.CreatedAt = t
|
||||||
|
awRepo.Data[h] = a
|
||||||
|
}
|
||||||
|
|
||||||
|
It("deletes orphan rows and their store files, keeps referenced ones", func() {
|
||||||
|
data := []byte("orphan-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||||
|
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
ageArtwork(h, old)
|
||||||
|
awRepo.OrphanHashes = []string{h}
|
||||||
|
|
||||||
|
kept := []byte("kept-bytes")
|
||||||
|
hk, _ := HashImage(bytes.NewReader(kept))
|
||||||
|
Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed())
|
||||||
|
Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||||
|
|
||||||
|
_, err := awRepo.GetImage(h)
|
||||||
|
Expect(err).To(MatchError(model.ErrNotFound))
|
||||||
|
_, err = store.Open(h, "image/jpeg")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
rc, err := store.Open(hk, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("spares a candidate reacquired between snapshot and delete", func() {
|
||||||
|
data := []byte("reacquired-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
ageArtwork(h, time.Now().Add(-2*time.Hour))
|
||||||
|
awRepo.OrphanHashes = []string{h}
|
||||||
|
// Reacquisition: an item now references the hash the snapshot flagged as orphan.
|
||||||
|
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
|
||||||
|
ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed())
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||||
|
|
||||||
|
_, err := awRepo.GetImage(h)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() {
|
||||||
|
data := []byte("fresh-reacquired-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
// Reacquisition refreshed created_at after the snapshot; still unreferenced.
|
||||||
|
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
awRepo.OrphanHashes = []string{h}
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||||
|
|
||||||
|
_, err := awRepo.GetImage(h)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("spares an orphan file freshly touched by an overlapping acquisition", func() {
|
||||||
|
data := []byte("racing-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
ageArtwork(h, time.Now().Add(-2*time.Hour))
|
||||||
|
awRepo.OrphanHashes = []string{h}
|
||||||
|
// The row is legitimately orphaned, but a concurrent acquisition just touched the
|
||||||
|
// file's mtime (duplicate Write) and is about to commit a row referencing it.
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||||
|
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("sweeps store files that have no artwork row", func() {
|
||||||
|
stray := []byte("no-row-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(stray))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||||
|
|
||||||
|
_, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("sweeps an obsolete mime variant of a reacquired hash", func() {
|
||||||
|
data := []byte("variant-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||||
|
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||||
|
// The row records the current mime; the .png file is a superseded variant.
|
||||||
|
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||||
|
|
||||||
|
_, err := store.Open(h, "image/png")
|
||||||
|
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
|
||||||
|
It("never sweeps files on a transient DB error", func() {
|
||||||
|
ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()}
|
||||||
|
|
||||||
|
data := []byte("live-bytes")
|
||||||
|
h, _ := HashImage(bytes.NewReader(data))
|
||||||
|
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||||
|
|
||||||
|
Expect(Prune(context.Background(), ds, store)).ToNot(Succeed())
|
||||||
|
|
||||||
|
rc, err := store.Open(h, "image/jpeg")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
rc.Close()
|
||||||
|
})
|
||||||
|
})
|
||||||
42
db/migrations/20260722023032_add_artwork_tables.sql
Normal file
42
db/migrations/20260722023032_add_artwork_tables.sql
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
-- +goose Up
|
||||||
|
CREATE TABLE artwork (
|
||||||
|
hash TEXT PRIMARY KEY,
|
||||||
|
mime TEXT NOT NULL,
|
||||||
|
width INTEGER NOT NULL DEFAULT 0,
|
||||||
|
height INTEGER NOT NULL DEFAULT 0,
|
||||||
|
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||||
|
blur_hash TEXT NOT NULL DEFAULT '',
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE item_artwork (
|
||||||
|
item_kind TEXT NOT NULL,
|
||||||
|
item_id TEXT NOT NULL,
|
||||||
|
image_type TEXT NOT NULL DEFAULT 'primary',
|
||||||
|
hash TEXT NOT NULL DEFAULT '',
|
||||||
|
source TEXT NOT NULL DEFAULT '',
|
||||||
|
source_path TEXT NOT NULL DEFAULT '',
|
||||||
|
ref_mtime INTEGER NOT NULL DEFAULT 0,
|
||||||
|
attempted_at TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP,
|
||||||
|
PRIMARY KEY (item_kind, item_id, image_type)
|
||||||
|
) WITHOUT ROWID;
|
||||||
|
CREATE INDEX ix_item_artwork_hash ON item_artwork(hash);
|
||||||
|
|
||||||
|
CREATE TABLE artwork_queue (
|
||||||
|
item_kind TEXT NOT NULL,
|
||||||
|
item_id TEXT NOT NULL,
|
||||||
|
image_type TEXT NOT NULL DEFAULT 'primary',
|
||||||
|
priority INTEGER NOT NULL DEFAULT 0,
|
||||||
|
attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
PRIMARY KEY (item_kind, item_id, image_type)
|
||||||
|
) WITHOUT ROWID;
|
||||||
|
-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering.
|
||||||
|
CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at);
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
DROP TABLE artwork_queue;
|
||||||
|
DROP TABLE item_artwork;
|
||||||
|
DROP TABLE artwork;
|
||||||
2
go.mod
2
go.mod
|
|
@ -57,6 +57,7 @@ require (
|
||||||
github.com/tetratelabs/wazero v1.12.0
|
github.com/tetratelabs/wazero v1.12.0
|
||||||
github.com/unrolled/secure v1.17.0
|
github.com/unrolled/secure v1.17.0
|
||||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
|
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
|
||||||
|
github.com/zeebo/xxh3 v1.1.0
|
||||||
go.senan.xyz/taglib v0.11.1
|
go.senan.xyz/taglib v0.11.1
|
||||||
go.uber.org/goleak v1.3.0
|
go.uber.org/goleak v1.3.0
|
||||||
golang.org/x/image v0.44.0
|
golang.org/x/image v0.44.0
|
||||||
|
|
@ -128,7 +129,6 @@ require (
|
||||||
github.com/subosito/gotenv v1.6.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
|
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
|
||||||
github.com/valyala/fastjson v1.6.10 // indirect
|
github.com/valyala/fastjson v1.6.10 // indirect
|
||||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
|
||||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||||
|
|
|
||||||
93
model/artwork.go
Normal file
93
model/artwork.go
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
package model
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// Artwork is one unique image, identified by the XXH3-64 hash of its bytes.
|
||||||
|
type Artwork struct {
|
||||||
|
Hash string `structs:"hash"`
|
||||||
|
Mime string `structs:"mime"`
|
||||||
|
Width int `structs:"width"`
|
||||||
|
Height int `structs:"height"`
|
||||||
|
SizeBytes int64 `structs:"size_bytes"`
|
||||||
|
BlurHash string `structs:"blur_hash"`
|
||||||
|
CreatedAt time.Time `structs:"created_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
const ImageTypePrimary = "primary"
|
||||||
|
|
||||||
|
// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent.
|
||||||
|
type ItemArtwork struct {
|
||||||
|
ItemKind string `structs:"item_kind"`
|
||||||
|
ItemID string `structs:"item_id"`
|
||||||
|
ImageType string `structs:"image_type"`
|
||||||
|
Hash string `structs:"hash"`
|
||||||
|
Source string `structs:"source"`
|
||||||
|
// SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise.
|
||||||
|
SourcePath string `structs:"source_path"`
|
||||||
|
// RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath.
|
||||||
|
RefMtime int64 `structs:"ref_mtime"`
|
||||||
|
// attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork;
|
||||||
|
// raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL.
|
||||||
|
AttemptedAt time.Time `structs:"attempted_at"`
|
||||||
|
UpdatedAt time.Time `structs:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork).
|
||||||
|
type ItemArtworkInfo struct {
|
||||||
|
ItemID string
|
||||||
|
Hash string
|
||||||
|
BlurHash string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Absent reports a known-absent artwork state (resolved, no image).
|
||||||
|
func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" }
|
||||||
|
|
||||||
|
type ArtworkQueueItem struct {
|
||||||
|
ItemKind string `structs:"item_kind"`
|
||||||
|
ItemID string `structs:"item_id"`
|
||||||
|
ImageType string `structs:"image_type"`
|
||||||
|
Priority int `structs:"priority"`
|
||||||
|
Attempts int `structs:"attempts"`
|
||||||
|
RetryAt time.Time `structs:"retry_at"`
|
||||||
|
EnqueuedAt time.Time `structs:"enqueued_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Queue priorities: higher drains first.
|
||||||
|
const (
|
||||||
|
ArtworkPriorityRecheck = 0
|
||||||
|
ArtworkPriorityBackfill = 10
|
||||||
|
ArtworkPriorityScan = 50
|
||||||
|
ArtworkPriorityBump = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
type ArtworkRepository interface {
|
||||||
|
// Image identity (artwork table)
|
||||||
|
GetImage(hash string) (*Artwork, error)
|
||||||
|
PutImage(a *Artwork) error
|
||||||
|
GetImages(hashes []string) (map[string]Artwork, error)
|
||||||
|
// GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff.
|
||||||
|
GetOrphanHashes(createdBefore time.Time) ([]string, error)
|
||||||
|
// DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check).
|
||||||
|
DeleteOrphans(createdBefore time.Time, hashes []string) error
|
||||||
|
// Per-item state (item_artwork table)
|
||||||
|
GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error)
|
||||||
|
PutItemArtwork(ia *ItemArtwork) error
|
||||||
|
DeleteForItem(kind, id string) error
|
||||||
|
// GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork.
|
||||||
|
GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error)
|
||||||
|
// GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks.
|
||||||
|
GetAllMimes() (map[string]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ArtworkQueueRepository interface {
|
||||||
|
// Enqueue upserts; an existing row keeps the higher of the two priorities.
|
||||||
|
Enqueue(items ...ArtworkQueueItem) error
|
||||||
|
// DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc.
|
||||||
|
DequeueBatch(n int) ([]ArtworkQueueItem, error)
|
||||||
|
// MarkFailed increments attempts and pushes retry_at into the future.
|
||||||
|
MarkFailed(kind, id, imageType string, retryAt time.Time) error
|
||||||
|
Delete(kind, id, imageType string) error
|
||||||
|
Count() (int64, error)
|
||||||
|
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.
|
||||||
|
EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error)
|
||||||
|
}
|
||||||
|
|
@ -40,6 +40,8 @@ type DataStore interface {
|
||||||
ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository
|
ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository
|
||||||
Scrobble(ctx context.Context) ScrobbleRepository
|
Scrobble(ctx context.Context) ScrobbleRepository
|
||||||
Plugin(ctx context.Context) PluginRepository
|
Plugin(ctx context.Context) PluginRepository
|
||||||
|
Artwork(ctx context.Context) ArtworkRepository
|
||||||
|
ArtworkQueue(ctx context.Context) ArtworkQueueRepository
|
||||||
|
|
||||||
Resource(ctx context.Context, model any) ResourceRepository
|
Resource(ctx context.Context, model any) ResourceRepository
|
||||||
|
|
||||||
|
|
|
||||||
90
persistence/artwork_queue_repository.go
Normal file
90
persistence/artwork_queue_repository.go
Normal file
|
|
@ -0,0 +1,90 @@
|
||||||
|
package persistence
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
. "github.com/Masterminds/squirrel"
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
)
|
||||||
|
|
||||||
|
// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars).
|
||||||
|
const enqueueChunkSize = 100
|
||||||
|
|
||||||
|
type artworkQueueRepository struct {
|
||||||
|
sqlRepository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository {
|
||||||
|
r := &artworkQueueRepository{}
|
||||||
|
r.ctx = ctx
|
||||||
|
r.db = db
|
||||||
|
r.tableName = "artwork_queue"
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||||
|
now := time.Now()
|
||||||
|
for chunk := range slices.Chunk(items, enqueueChunkSize) {
|
||||||
|
ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at")
|
||||||
|
for _, it := range chunk {
|
||||||
|
if it.ImageType == "" {
|
||||||
|
it.ImageType = model.ImageTypePrimary
|
||||||
|
}
|
||||||
|
ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now)
|
||||||
|
}
|
||||||
|
ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
|
||||||
|
priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`)
|
||||||
|
if _, err := r.executeSQL(ins); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||||
|
sel := Select("*").From(r.tableName).
|
||||||
|
Where(LtOrEq{"retry_at": time.Now()}).
|
||||||
|
OrderBy("priority DESC", "enqueued_at ASC").
|
||||||
|
Limit(uint64(n))
|
||||||
|
var res []model.ArtworkQueueItem
|
||||||
|
err := r.queryAll(sel, &res)
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
|
||||||
|
upd := Update(r.tableName).
|
||||||
|
Set("attempts", Expr("attempts + 1")).
|
||||||
|
Set("retry_at", retryAt).
|
||||||
|
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||||
|
c, err := r.executeSQL(upd)
|
||||||
|
if err == nil && c == 0 {
|
||||||
|
return model.ErrNotFound
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkQueueRepository) Delete(kind, id, imageType string) error {
|
||||||
|
return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkQueueRepository) Count() (int64, error) {
|
||||||
|
var res struct{ Count int64 }
|
||||||
|
err := r.queryOne(Select("count(*) as count").From(r.tableName), &res)
|
||||||
|
return res.Count, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
|
||||||
|
now := time.Now()
|
||||||
|
// DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items.
|
||||||
|
ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at)
|
||||||
|
SELECT item_kind, item_id, image_type, ?, 0, ?, ?
|
||||||
|
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
|
||||||
|
ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`,
|
||||||
|
model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore)
|
||||||
|
return r.executeSQL(ins)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil)
|
||||||
83
persistence/artwork_queue_repository_test.go
Normal file
83
persistence/artwork_queue_repository_test.go
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
package persistence
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
. "github.com/onsi/ginkgo/v2"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
)
|
||||||
|
|
||||||
|
var _ = Describe("ArtworkQueueRepository", func() {
|
||||||
|
var repo model.ArtworkQueueRepository
|
||||||
|
|
||||||
|
item := func(kind, id string, prio int) model.ArtworkQueueItem {
|
||||||
|
return model.ArtworkQueueItem{ItemKind: kind, ItemID: id,
|
||||||
|
ImageType: model.ImageTypePrimary, Priority: prio}
|
||||||
|
}
|
||||||
|
|
||||||
|
BeforeEach(func() {
|
||||||
|
clearArtworkTables()
|
||||||
|
repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("enqueues and dequeues by priority then FIFO", func() {
|
||||||
|
Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed())
|
||||||
|
Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed())
|
||||||
|
|
||||||
|
got, err := repo.DequeueBatch(10)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got).To(HaveLen(2))
|
||||||
|
Expect(got[0].ItemID).To(Equal("high"))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("keeps the higher priority on duplicate enqueue", func() {
|
||||||
|
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed())
|
||||||
|
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed())
|
||||||
|
got, _ := repo.DequeueBatch(10)
|
||||||
|
Expect(got).To(HaveLen(1))
|
||||||
|
Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("hides failed items until retry_at", func() {
|
||||||
|
Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed())
|
||||||
|
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed())
|
||||||
|
|
||||||
|
got, err := repo.DequeueBatch(10)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got).To(BeEmpty())
|
||||||
|
|
||||||
|
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed())
|
||||||
|
got, _ = repo.DequeueBatch(10)
|
||||||
|
Expect(got).To(HaveLen(1))
|
||||||
|
Expect(got[0].Attempts).To(Equal(2))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("deletes on completion and counts", func() {
|
||||||
|
Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed())
|
||||||
|
n, _ := repo.Count()
|
||||||
|
Expect(n).To(Equal(int64(1)))
|
||||||
|
Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed())
|
||||||
|
n, _ = repo.Count()
|
||||||
|
Expect(n).To(BeZero())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("enqueues stale absent states for recheck", func() {
|
||||||
|
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||||
|
old := time.Now().Add(-48 * time.Hour)
|
||||||
|
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed())
|
||||||
|
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||||
|
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
|
||||||
|
|
||||||
|
n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour))
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(n).To(Equal(int64(1)))
|
||||||
|
|
||||||
|
items, err := repo.DequeueBatch(10)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(items).To(HaveLen(1))
|
||||||
|
Expect(items[0].ItemID).To(Equal("stale1"))
|
||||||
|
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
|
||||||
|
})
|
||||||
|
})
|
||||||
179
persistence/artwork_repository.go
Normal file
179
persistence/artwork_repository.go
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
package persistence
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
. "github.com/Masterminds/squirrel"
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
itemArtworkTable = "item_artwork"
|
||||||
|
artworkBatchSize = 200
|
||||||
|
)
|
||||||
|
|
||||||
|
type itemArtworkSQL struct {
|
||||||
|
sqlRepository
|
||||||
|
}
|
||||||
|
|
||||||
|
type artworkRepository struct {
|
||||||
|
sqlRepository
|
||||||
|
items itemArtworkSQL
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository {
|
||||||
|
r := &artworkRepository{}
|
||||||
|
r.ctx = ctx
|
||||||
|
r.db = db
|
||||||
|
r.tableName = "artwork"
|
||||||
|
r.items.ctx = ctx
|
||||||
|
r.items.db = db
|
||||||
|
r.items.tableName = itemArtworkTable
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) {
|
||||||
|
sel := Select("*").From(r.tableName).Where(Eq{"hash": hash})
|
||||||
|
var res model.Artwork
|
||||||
|
if err := r.queryOne(sel, &res); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) PutImage(a *model.Artwork) error {
|
||||||
|
// created_at is the last-acquisition-write time the prune grace window keys on.
|
||||||
|
a.CreatedAt = time.Now()
|
||||||
|
values, err := toSQLArgs(*a)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window.
|
||||||
|
ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width,
|
||||||
|
height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`)
|
||||||
|
_, err = r.executeSQL(ins)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) {
|
||||||
|
res := map[string]model.Artwork{}
|
||||||
|
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
|
||||||
|
sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk})
|
||||||
|
var all []model.Artwork
|
||||||
|
if err := r.queryAll(sel, &all); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, a := range all {
|
||||||
|
res[a.Hash] = a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) GetAllMimes() (map[string]string, error) {
|
||||||
|
sel := Select("hash", "mime").From(r.tableName)
|
||||||
|
var rows []struct {
|
||||||
|
Hash string
|
||||||
|
Mime string
|
||||||
|
}
|
||||||
|
if err := r.queryAll(sel, &rows); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
res := make(map[string]string, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
res[row.Hash] = row.Mime
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
|
||||||
|
sel := Select("hash").From(r.tableName).
|
||||||
|
Where(And{
|
||||||
|
Lt{"created_at": createdBefore},
|
||||||
|
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
|
||||||
|
})
|
||||||
|
var hashes []string
|
||||||
|
err := r.queryAllSlice(sel, &hashes)
|
||||||
|
return hashes, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error {
|
||||||
|
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
|
||||||
|
del := Delete(r.tableName).Where(And{
|
||||||
|
Eq{"hash": chunk},
|
||||||
|
Lt{"created_at": createdBefore},
|
||||||
|
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
|
||||||
|
})
|
||||||
|
if _, err := r.executeSQL(del); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
|
||||||
|
sel := Select("*").From(itemArtworkTable).
|
||||||
|
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||||
|
var res model.ItemArtwork
|
||||||
|
if err := r.items.queryOne(sel, &res); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error {
|
||||||
|
if ia.ImageType == "" {
|
||||||
|
ia.ImageType = model.ImageTypePrimary
|
||||||
|
}
|
||||||
|
ia.UpdatedAt = time.Now()
|
||||||
|
// PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now.
|
||||||
|
if ia.AttemptedAt.IsZero() {
|
||||||
|
ia.AttemptedAt = ia.UpdatedAt
|
||||||
|
}
|
||||||
|
values, err := toSQLArgs(*ia)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
|
||||||
|
hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime,
|
||||||
|
attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`)
|
||||||
|
_, err = r.items.executeSQL(ins)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) DeleteForItem(kind, id string) error {
|
||||||
|
return r.items.delete(Eq{"item_kind": kind, "item_id": id})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
|
||||||
|
res := map[string]model.ItemArtworkInfo{}
|
||||||
|
for chunk := range slices.Chunk(ids, artworkBatchSize) {
|
||||||
|
sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash").
|
||||||
|
From(itemArtworkTable + " ia").
|
||||||
|
LeftJoin("artwork a ON a.hash = ia.hash").
|
||||||
|
Where(And{
|
||||||
|
Eq{"ia.item_kind": kind},
|
||||||
|
Eq{"ia.image_type": model.ImageTypePrimary},
|
||||||
|
Eq{"ia.item_id": chunk},
|
||||||
|
})
|
||||||
|
var rows []struct {
|
||||||
|
ItemID string
|
||||||
|
Hash string
|
||||||
|
BlurHash string
|
||||||
|
}
|
||||||
|
if err := r.items.queryAll(sel, &rows); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
res[row.ItemID] = model.ItemArtworkInfo{
|
||||||
|
ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ model.ArtworkRepository = (*artworkRepository)(nil)
|
||||||
197
persistence/artwork_repository_test.go
Normal file
197
persistence/artwork_repository_test.go
Normal file
|
|
@ -0,0 +1,197 @@
|
||||||
|
package persistence
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
. "github.com/onsi/ginkgo/v2"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
)
|
||||||
|
|
||||||
|
// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state.
|
||||||
|
func clearArtworkTables() {
|
||||||
|
db := GetDBXBuilder()
|
||||||
|
for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} {
|
||||||
|
_, err := db.NewQuery("DELETE FROM " + t).Execute()
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ = Describe("ArtworkRepository", func() {
|
||||||
|
var repo model.ArtworkRepository
|
||||||
|
|
||||||
|
BeforeEach(func() {
|
||||||
|
clearArtworkTables()
|
||||||
|
repo = NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||||
|
})
|
||||||
|
|
||||||
|
Context("image identity", func() {
|
||||||
|
It("stores and retrieves an artwork by hash", func() {
|
||||||
|
a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"}
|
||||||
|
Expect(repo.PutImage(a)).To(Succeed())
|
||||||
|
|
||||||
|
got, err := repo.GetImage("abc123")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got.Mime).To(Equal("image/jpeg"))
|
||||||
|
Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w"))
|
||||||
|
Expect(got.CreatedAt).ToNot(BeZero())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("is idempotent on Put (upsert by hash)", func() {
|
||||||
|
a := &model.Artwork{Hash: "dup1", Mime: "image/png"}
|
||||||
|
Expect(repo.PutImage(a)).To(Succeed())
|
||||||
|
a.BlurHash = "XYZ"
|
||||||
|
Expect(repo.PutImage(a)).To(Succeed())
|
||||||
|
got, _ := repo.GetImage("dup1")
|
||||||
|
Expect(got.BlurHash).To(Equal("XYZ"))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("refreshes created_at when reacquiring an existing hash", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
_, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'").
|
||||||
|
Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute()
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed())
|
||||||
|
|
||||||
|
got, err := repo.GetImage("reacq")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("returns ErrNotFound for a missing hash", func() {
|
||||||
|
_, err := repo.GetImage("nope")
|
||||||
|
Expect(err).To(MatchError(model.ErrNotFound))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("fetches a batch", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed())
|
||||||
|
got, err := repo.GetImages([]string{"b1", "b2", "missing"})
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got).To(HaveLen(2))
|
||||||
|
Expect(got["b2"].Mime).To(Equal("image/png"))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("returns every stored hash with its current mime", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed())
|
||||||
|
mimes, err := repo.GetAllMimes()
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg"))
|
||||||
|
Expect(mimes).To(HaveKeyWithValue("all2", "image/png"))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("finds orphans older than cutoff, honoring item_artwork references", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed())
|
||||||
|
|
||||||
|
orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute))
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(orphans).To(ContainElement("orph1"))
|
||||||
|
Expect(orphans).ToNot(ContainElement("ref1"))
|
||||||
|
|
||||||
|
orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour))
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(orphans).To(BeEmpty())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("deletes only unreferenced hashes older than the cutoff", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
|
||||||
|
ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed())
|
||||||
|
|
||||||
|
Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed())
|
||||||
|
|
||||||
|
_, err := repo.GetImage("d1")
|
||||||
|
Expect(err).To(MatchError(model.ErrNotFound))
|
||||||
|
_, err = repo.GetImage("dref")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("spares an unreferenced hash younger than the cutoff", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed())
|
||||||
|
_, err := repo.GetImage("young")
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("fetches a batch larger than the SQL variable limit", func() {
|
||||||
|
hashes := make([]string, 0, 250)
|
||||||
|
for i := range 250 {
|
||||||
|
h := fmt.Sprintf("big%03d", i)
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||||
|
hashes = append(hashes, h)
|
||||||
|
}
|
||||||
|
hashes = append(hashes, "absent1", "absent2")
|
||||||
|
|
||||||
|
got, err := repo.GetImages(hashes)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got).To(HaveLen(250))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
Context("item state", func() {
|
||||||
|
It("upserts and reads state, including per-item provenance", func() {
|
||||||
|
ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary,
|
||||||
|
Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()}
|
||||||
|
Expect(repo.PutItemArtwork(ia)).To(Succeed())
|
||||||
|
ia.Source = "embedded"
|
||||||
|
ia.SourcePath = "/music/a/track.mp3"
|
||||||
|
ia.RefMtime = 222
|
||||||
|
Expect(repo.PutItemArtwork(ia)).To(Succeed())
|
||||||
|
|
||||||
|
got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got.Source).To(Equal("embedded"))
|
||||||
|
Expect(got.SourcePath).To(Equal("/music/a/track.mp3"))
|
||||||
|
Expect(got.RefMtime).To(Equal(int64(222)))
|
||||||
|
Expect(got.UpdatedAt).ToNot(BeZero())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("defaults attempted_at to now when unset", func() {
|
||||||
|
before := time.Now().Add(-time.Second)
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt",
|
||||||
|
ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed())
|
||||||
|
got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got.AttemptedAt).To(BeTemporally(">", before))
|
||||||
|
})
|
||||||
|
|
||||||
|
It("represents known-absent as empty hash", func() {
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1",
|
||||||
|
ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||||
|
got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(got.Hash).To(BeEmpty())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("hydrates a page in one batch, including blurhash and absence", func() {
|
||||||
|
Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed())
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed())
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed())
|
||||||
|
|
||||||
|
info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"})
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
Expect(info).To(HaveLen(2))
|
||||||
|
Expect(info["x1"].Hash).To(Equal("h9"))
|
||||||
|
Expect(info["x1"].BlurHash).To(Equal("BH9"))
|
||||||
|
Expect(info["x1"].Absent()).To(BeFalse())
|
||||||
|
Expect(info["x2"].Absent()).To(BeTrue())
|
||||||
|
_, unresolved := info["x3"]
|
||||||
|
Expect(unresolved).To(BeFalse())
|
||||||
|
})
|
||||||
|
|
||||||
|
It("deletes all rows for an item", func() {
|
||||||
|
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed())
|
||||||
|
Expect(repo.DeleteForItem("pl", "p1")).To(Succeed())
|
||||||
|
_, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary)
|
||||||
|
Expect(err).To(MatchError(model.ErrNotFound))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -97,6 +97,14 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository {
|
||||||
return NewPluginRepository(ctx, s.getDBXBuilder())
|
return NewPluginRepository(ctx, s.getDBXBuilder())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository {
|
||||||
|
return NewArtworkRepository(ctx, s.getDBXBuilder())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
|
||||||
|
return NewArtworkQueueRepository(ctx, s.getDBXBuilder())
|
||||||
|
}
|
||||||
|
|
||||||
func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository {
|
func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository {
|
||||||
switch m.(type) {
|
switch m.(type) {
|
||||||
case model.User:
|
case model.User:
|
||||||
|
|
|
||||||
125
tests/mock_artwork_queue_repo.go
Normal file
125
tests/mock_artwork_queue_repo.go
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
package tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MockArtworkQueueRepo struct {
|
||||||
|
model.ArtworkQueueRepository
|
||||||
|
Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType)
|
||||||
|
Err error
|
||||||
|
// ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state.
|
||||||
|
ItemArtworkSource *MockArtworkRepo
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo {
|
||||||
|
return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
for _, it := range items {
|
||||||
|
if it.ImageType == "" {
|
||||||
|
it.ImageType = model.ImageTypePrimary
|
||||||
|
}
|
||||||
|
k := iaKey(it.ItemKind, it.ItemID, it.ImageType)
|
||||||
|
// Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller.
|
||||||
|
if prev, ok := m.Data[k]; ok {
|
||||||
|
prev.Priority = max(prev.Priority, it.Priority)
|
||||||
|
prev.RetryAt = now
|
||||||
|
m.Data[k] = prev
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
it.Attempts = 0
|
||||||
|
it.RetryAt = now
|
||||||
|
it.EnqueuedAt = now
|
||||||
|
m.Data[k] = it
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return nil, m.Err
|
||||||
|
}
|
||||||
|
var res []model.ArtworkQueueItem
|
||||||
|
now := time.Now()
|
||||||
|
for _, it := range m.Data {
|
||||||
|
if !it.RetryAt.After(now) {
|
||||||
|
res = append(res, it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(res, func(i, j int) bool {
|
||||||
|
if res[i].Priority != res[j].Priority {
|
||||||
|
return res[i].Priority > res[j].Priority
|
||||||
|
}
|
||||||
|
return res[i].EnqueuedAt.Before(res[j].EnqueuedAt)
|
||||||
|
})
|
||||||
|
if len(res) > n {
|
||||||
|
res = res[:n]
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
k := iaKey(kind, id, imageType)
|
||||||
|
it, ok := m.Data[k]
|
||||||
|
if !ok {
|
||||||
|
return model.ErrNotFound
|
||||||
|
}
|
||||||
|
it.Attempts++
|
||||||
|
it.RetryAt = retryAt
|
||||||
|
m.Data[k] = it
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
delete(m.Data, iaKey(kind, id, imageType))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkQueueRepo) Count() (int64, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return 0, m.Err
|
||||||
|
}
|
||||||
|
return int64(len(m.Data)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
|
||||||
|
if m.Err != nil || m.ItemArtworkSource == nil {
|
||||||
|
return 0, m.Err
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
var inserted int64
|
||||||
|
for _, ia := range m.ItemArtworkSource.ItemData {
|
||||||
|
if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)
|
||||||
|
if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.Data[k] = model.ArtworkQueueItem{
|
||||||
|
ItemKind: ia.ItemKind,
|
||||||
|
ItemID: ia.ItemID,
|
||||||
|
ImageType: ia.ImageType,
|
||||||
|
Priority: model.ArtworkPriorityRecheck,
|
||||||
|
RetryAt: now,
|
||||||
|
EnqueuedAt: now,
|
||||||
|
}
|
||||||
|
inserted++
|
||||||
|
}
|
||||||
|
return inserted, nil
|
||||||
|
}
|
||||||
149
tests/mock_artwork_repo.go
Normal file
149
tests/mock_artwork_repo.go
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
package tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/navidrome/navidrome/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MockArtworkRepo struct {
|
||||||
|
model.ArtworkRepository
|
||||||
|
Data map[string]model.Artwork
|
||||||
|
ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType)
|
||||||
|
OrphanHashes []string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateMockArtworkRepo() *MockArtworkRepo {
|
||||||
|
return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType }
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return nil, m.Err
|
||||||
|
}
|
||||||
|
if a, ok := m.Data[hash]; ok {
|
||||||
|
return &a, nil
|
||||||
|
}
|
||||||
|
return nil, model.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) PutImage(a *model.Artwork) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
// Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly.
|
||||||
|
a.CreatedAt = time.Now()
|
||||||
|
m.Data[a.Hash] = *a
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return nil, m.Err
|
||||||
|
}
|
||||||
|
res := map[string]model.Artwork{}
|
||||||
|
for _, h := range hashes {
|
||||||
|
if a, ok := m.Data[h]; ok {
|
||||||
|
res[h] = a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
|
||||||
|
return m.OrphanHashes, m.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return nil, m.Err
|
||||||
|
}
|
||||||
|
mimes := make(map[string]string, len(m.Data))
|
||||||
|
for h, a := range m.Data {
|
||||||
|
mimes[h] = a.Mime
|
||||||
|
}
|
||||||
|
return mimes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
// Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted.
|
||||||
|
for _, h := range hashes {
|
||||||
|
if m.referenced(h) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(m.Data, h)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) referenced(hash string) bool {
|
||||||
|
for _, ia := range m.ItemData {
|
||||||
|
if ia.Hash == hash {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return nil, m.Err
|
||||||
|
}
|
||||||
|
if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok {
|
||||||
|
return &ia, nil
|
||||||
|
}
|
||||||
|
return nil, model.ErrNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
if ia.ImageType == "" {
|
||||||
|
ia.ImageType = model.ImageTypePrimary
|
||||||
|
}
|
||||||
|
ia.UpdatedAt = time.Now()
|
||||||
|
if ia.AttemptedAt.IsZero() {
|
||||||
|
ia.AttemptedAt = ia.UpdatedAt
|
||||||
|
}
|
||||||
|
m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) DeleteForItem(kind, id string) error {
|
||||||
|
if m.Err != nil {
|
||||||
|
return m.Err
|
||||||
|
}
|
||||||
|
for k, ia := range m.ItemData {
|
||||||
|
if ia.ItemKind == kind && ia.ItemID == id {
|
||||||
|
delete(m.ItemData, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
|
||||||
|
if m.Err != nil {
|
||||||
|
return nil, m.Err
|
||||||
|
}
|
||||||
|
res := map[string]model.ItemArtworkInfo{}
|
||||||
|
for _, id := range ids {
|
||||||
|
if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok {
|
||||||
|
info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash}
|
||||||
|
if a, ok := m.Data[ia.Hash]; ok {
|
||||||
|
info.BlurHash = a.BlurHash
|
||||||
|
}
|
||||||
|
res[id] = info
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
@ -28,6 +28,8 @@ type MockDataStore struct {
|
||||||
MockedScrobble model.ScrobbleRepository
|
MockedScrobble model.ScrobbleRepository
|
||||||
MockedRadio model.RadioRepository
|
MockedRadio model.RadioRepository
|
||||||
MockedPlugin model.PluginRepository
|
MockedPlugin model.PluginRepository
|
||||||
|
MockedArtwork model.ArtworkRepository
|
||||||
|
MockedArtworkQueue model.ArtworkQueueRepository
|
||||||
scrobbleBufferMu sync.Mutex
|
scrobbleBufferMu sync.Mutex
|
||||||
repoMu sync.Mutex
|
repoMu sync.Mutex
|
||||||
|
|
||||||
|
|
@ -247,6 +249,32 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository {
|
||||||
return db.MockedPlugin
|
return db.MockedPlugin
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository {
|
||||||
|
if db.MockedArtwork != nil {
|
||||||
|
return db.MockedArtwork
|
||||||
|
}
|
||||||
|
if db.RealDS != nil {
|
||||||
|
return db.RealDS.Artwork(ctx)
|
||||||
|
}
|
||||||
|
db.MockedArtwork = CreateMockArtworkRepo()
|
||||||
|
return db.MockedArtwork
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
|
||||||
|
if db.MockedArtworkQueue != nil {
|
||||||
|
return db.MockedArtworkQueue
|
||||||
|
}
|
||||||
|
if db.RealDS != nil {
|
||||||
|
return db.RealDS.ArtworkQueue(ctx)
|
||||||
|
}
|
||||||
|
q := CreateMockArtworkQueueRepo()
|
||||||
|
if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok {
|
||||||
|
q.ItemArtworkSource = aw
|
||||||
|
}
|
||||||
|
db.MockedArtworkQueue = q
|
||||||
|
return db.MockedArtworkQueue
|
||||||
|
}
|
||||||
|
|
||||||
func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error {
|
func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error {
|
||||||
return block(db)
|
return block(db)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue