navidrome/utils/cache/spread_fs_test.go

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

131 lines
3.8 KiB
Go
Raw Normal View History

2021-04-22 14:02:42 -04:00
package cache
import (
"os"
"path/filepath"
"strings"
2022-07-26 16:47:16 -04:00
. "github.com/onsi/ginkgo/v2"
2021-04-22 14:02:42 -04:00
. "github.com/onsi/gomega"
)
var _ = Describe("Spread FS", func() {
var fs *spreadFS
var rootDir string
BeforeEach(func() {
var err error
rootDir, _ = os.MkdirTemp("", "spread_fs")
2021-04-22 14:02:42 -04:00
fs, err = NewSpreadFS(rootDir, 0755)
Expect(err).To(BeNil())
})
AfterEach(func() {
2021-10-28 10:40:31 -04:00
_ = os.RemoveAll(rootDir)
2021-04-22 14:02:42 -04:00
})
Describe("KeyMapper", func() {
It("creates a file with proper name format", func() {
mapped := fs.KeyMapper("abc")
Expect(mapped).To(HavePrefix(fs.root))
mapped = strings.TrimPrefix(mapped, fs.root)
parts := strings.Split(mapped, string(filepath.Separator))
Expect(parts).To(HaveLen(4))
Expect(parts[3]).To(HaveLen(40))
})
2022-07-25 23:34:09 -04:00
It("returns the unmodified key if it is a cache file path", func() {
mapped := fs.KeyMapper("abc")
Expect(mapped).To(HavePrefix(fs.root))
Expect(fs.KeyMapper(mapped)).To(Equal(mapped))
})
2021-04-22 14:02:42 -04:00
})
fix(cache): don't serve partially-written transcodes after a crash (#5657) * feat(cache): add completion marker helpers to spreadFS * feat(cache): write completion marker after successful cache write * fix(cache): adopt only complete files on reload, grandfather existing caches * test(cache): regression tests for partial-transcode crash leftover (#5636) * test(cache): guard concurrent in-progress streaming with completion marker * test(cache): make concurrent-streaming guard actually attach a second reader mid-write The previous test obtained s2 only after pw.Close(), so no reader ever attached to the in-progress entry. Now pw.Write("hello ") is called synchronously before the second Get — io.Pipe's blocking write gives a deterministic happens-before — then both s1 and s2 are drained in parallel goroutines while the producer writes the rest and closes the pipe. * style(cache): clarify best-effort intent of cleanup os.Remove calls * refactor(cache): lift one-time grandfather pass out of Reload's steady-state loop * refactor(cache): have MarkComplete take the key, owning path mapping in spreadFS * test(cache): assert no completion marker is written when the write fails * refactor(cache): rename migration sentinel to generic .nd-migrated * refactor(cache): rename grandfather migration to migrateExistingFiles * refactor(cache): single-pass Reload with safer marker-error handling Address PR review feedback: - Merge the one-time migration into Reload's single directory walk, avoiding a second full walk on first boot. - Only delete a data file when its marker is definitively absent (os.IsNotExist); skip on other stat errors to avoid destroying valid entries under transient I/O failures. - Write the migration sentinel only after a clean walk, so a partial walk can't strand valid-but-unmarked files for later deletion. - Return early from walkDataFiles on a WalkDir error. - Assert fs.Create error in the marker-removal test.
2026-06-23 22:23:54 -04:00
Describe("MarkComplete / Remove markers", func() {
It("creates a .complete marker for a data file", func() {
data := fs.KeyMapper("song1")
f, err := fs.Create(data)
Expect(err).To(BeNil())
_, _ = f.Write([]byte("ok"))
_ = f.Close()
Expect(fs.MarkComplete(data)).To(Succeed())
_, statErr := os.Stat(data + ".complete")
Expect(statErr).To(BeNil())
})
It("removes the sibling marker when the data file is removed", func() {
data := fs.KeyMapper("song2")
f, err := fs.Create(data)
Expect(err).To(BeNil())
_, _ = f.Write([]byte("ok"))
_ = f.Close()
Expect(fs.MarkComplete(data)).To(Succeed())
Expect(fs.Remove(data)).To(Succeed())
_, dataErr := os.Stat(data)
Expect(os.IsNotExist(dataErr)).To(BeTrue())
_, markErr := os.Stat(data + ".complete")
Expect(os.IsNotExist(markErr)).To(BeTrue())
})
})
2021-04-22 14:02:42 -04:00
Describe("Reload", func() {
fix(cache): don't serve partially-written transcodes after a crash (#5657) * feat(cache): add completion marker helpers to spreadFS * feat(cache): write completion marker after successful cache write * fix(cache): adopt only complete files on reload, grandfather existing caches * test(cache): regression tests for partial-transcode crash leftover (#5636) * test(cache): guard concurrent in-progress streaming with completion marker * test(cache): make concurrent-streaming guard actually attach a second reader mid-write The previous test obtained s2 only after pw.Close(), so no reader ever attached to the in-progress entry. Now pw.Write("hello ") is called synchronously before the second Get — io.Pipe's blocking write gives a deterministic happens-before — then both s1 and s2 are drained in parallel goroutines while the producer writes the rest and closes the pipe. * style(cache): clarify best-effort intent of cleanup os.Remove calls * refactor(cache): lift one-time grandfather pass out of Reload's steady-state loop * refactor(cache): have MarkComplete take the key, owning path mapping in spreadFS * test(cache): assert no completion marker is written when the write fails * refactor(cache): rename migration sentinel to generic .nd-migrated * refactor(cache): rename grandfather migration to migrateExistingFiles * refactor(cache): single-pass Reload with safer marker-error handling Address PR review feedback: - Merge the one-time migration into Reload's single directory walk, avoiding a second full walk on first boot. - Only delete a data file when its marker is definitively absent (os.IsNotExist); skip on other stat errors to avoid destroying valid entries under transient I/O failures. - Write the migration sentinel only after a clean walk, so a partial walk can't strand valid-but-unmarked files for later deletion. - Return early from walkDataFiles on a WalkDir error. - Assert fs.Create error in the marker-removal test.
2026-06-23 22:23:54 -04:00
makeData := func(content string) string {
file := fs.KeyMapper(content)
f, err := fs.Create(file)
Expect(err).To(BeNil())
_, _ = f.Write([]byte(content))
_ = f.Close()
return file
}
It("migrates all existing files on first run and writes the sentinel", func() {
for _, c := range []string{"aaaaa", "bbbbb", "ccccc"} {
makeData(c) // no markers, simulating a pre-upgrade cache
2021-04-22 14:02:42 -04:00
}
var actual []string
fix(cache): don't serve partially-written transcodes after a crash (#5657) * feat(cache): add completion marker helpers to spreadFS * feat(cache): write completion marker after successful cache write * fix(cache): adopt only complete files on reload, grandfather existing caches * test(cache): regression tests for partial-transcode crash leftover (#5636) * test(cache): guard concurrent in-progress streaming with completion marker * test(cache): make concurrent-streaming guard actually attach a second reader mid-write The previous test obtained s2 only after pw.Close(), so no reader ever attached to the in-progress entry. Now pw.Write("hello ") is called synchronously before the second Get — io.Pipe's blocking write gives a deterministic happens-before — then both s1 and s2 are drained in parallel goroutines while the producer writes the rest and closes the pipe. * style(cache): clarify best-effort intent of cleanup os.Remove calls * refactor(cache): lift one-time grandfather pass out of Reload's steady-state loop * refactor(cache): have MarkComplete take the key, owning path mapping in spreadFS * test(cache): assert no completion marker is written when the write fails * refactor(cache): rename migration sentinel to generic .nd-migrated * refactor(cache): rename grandfather migration to migrateExistingFiles * refactor(cache): single-pass Reload with safer marker-error handling Address PR review feedback: - Merge the one-time migration into Reload's single directory walk, avoiding a second full walk on first boot. - Only delete a data file when its marker is definitively absent (os.IsNotExist); skip on other stat errors to avoid destroying valid entries under transient I/O failures. - Write the migration sentinel only after a clean walk, so a partial walk can't strand valid-but-unmarked files for later deletion. - Return early from walkDataFiles on a WalkDir error. - Assert fs.Create error in the marker-removal test.
2026-06-23 22:23:54 -04:00
err := fs.Reload(func(key, name string) {
2021-04-22 14:02:42 -04:00
Expect(key).To(Equal(name))
fix(cache): don't serve partially-written transcodes after a crash (#5657) * feat(cache): add completion marker helpers to spreadFS * feat(cache): write completion marker after successful cache write * fix(cache): adopt only complete files on reload, grandfather existing caches * test(cache): regression tests for partial-transcode crash leftover (#5636) * test(cache): guard concurrent in-progress streaming with completion marker * test(cache): make concurrent-streaming guard actually attach a second reader mid-write The previous test obtained s2 only after pw.Close(), so no reader ever attached to the in-progress entry. Now pw.Write("hello ") is called synchronously before the second Get — io.Pipe's blocking write gives a deterministic happens-before — then both s1 and s2 are drained in parallel goroutines while the producer writes the rest and closes the pipe. * style(cache): clarify best-effort intent of cleanup os.Remove calls * refactor(cache): lift one-time grandfather pass out of Reload's steady-state loop * refactor(cache): have MarkComplete take the key, owning path mapping in spreadFS * test(cache): assert no completion marker is written when the write fails * refactor(cache): rename migration sentinel to generic .nd-migrated * refactor(cache): rename grandfather migration to migrateExistingFiles * refactor(cache): single-pass Reload with safer marker-error handling Address PR review feedback: - Merge the one-time migration into Reload's single directory walk, avoiding a second full walk on first boot. - Only delete a data file when its marker is definitively absent (os.IsNotExist); skip on other stat errors to avoid destroying valid entries under transient I/O failures. - Write the migration sentinel only after a clean walk, so a partial walk can't strand valid-but-unmarked files for later deletion. - Return early from walkDataFiles on a WalkDir error. - Assert fs.Create error in the marker-removal test.
2026-06-23 22:23:54 -04:00
data, _ := os.ReadFile(name)
2021-04-22 14:02:42 -04:00
actual = append(actual, string(data))
})
Expect(err).To(BeNil())
fix(cache): don't serve partially-written transcodes after a crash (#5657) * feat(cache): add completion marker helpers to spreadFS * feat(cache): write completion marker after successful cache write * fix(cache): adopt only complete files on reload, grandfather existing caches * test(cache): regression tests for partial-transcode crash leftover (#5636) * test(cache): guard concurrent in-progress streaming with completion marker * test(cache): make concurrent-streaming guard actually attach a second reader mid-write The previous test obtained s2 only after pw.Close(), so no reader ever attached to the in-progress entry. Now pw.Write("hello ") is called synchronously before the second Get — io.Pipe's blocking write gives a deterministic happens-before — then both s1 and s2 are drained in parallel goroutines while the producer writes the rest and closes the pipe. * style(cache): clarify best-effort intent of cleanup os.Remove calls * refactor(cache): lift one-time grandfather pass out of Reload's steady-state loop * refactor(cache): have MarkComplete take the key, owning path mapping in spreadFS * test(cache): assert no completion marker is written when the write fails * refactor(cache): rename migration sentinel to generic .nd-migrated * refactor(cache): rename grandfather migration to migrateExistingFiles * refactor(cache): single-pass Reload with safer marker-error handling Address PR review feedback: - Merge the one-time migration into Reload's single directory walk, avoiding a second full walk on first boot. - Only delete a data file when its marker is definitively absent (os.IsNotExist); skip on other stat errors to avoid destroying valid entries under transient I/O failures. - Write the migration sentinel only after a clean walk, so a partial walk can't strand valid-but-unmarked files for later deletion. - Return early from walkDataFiles on a WalkDir error. - Assert fs.Create error in the marker-removal test.
2026-06-23 22:23:54 -04:00
Expect(actual).To(ContainElements("aaaaa", "bbbbb", "ccccc"))
Expect(actual).To(HaveLen(3))
_, sentinelErr := os.Stat(filepath.Join(rootDir, ".nd-migrated"))
Expect(sentinelErr).To(BeNil())
})
It("after migration, adopts only marked files and deletes unmarked partials", func() {
// Pretend migration already happened.
Expect(os.WriteFile(filepath.Join(rootDir, ".nd-migrated"), nil, 0600)).To(Succeed())
good := makeData("good")
Expect(fs.MarkComplete(good)).To(Succeed())
bad := makeData("bad") // partial: no marker
var actual []string
err := fs.Reload(func(key, name string) { actual = append(actual, name) })
Expect(err).To(BeNil())
Expect(actual).To(ConsistOf(good))
_, badErr := os.Stat(bad)
Expect(os.IsNotExist(badErr)).To(BeTrue()) // partial deleted
})
It("ignores and cleans orphan markers", func() {
Expect(os.WriteFile(filepath.Join(rootDir, ".nd-migrated"), nil, 0600)).To(Succeed())
orphan := fs.KeyMapper("orphan") + ".complete"
Expect(os.MkdirAll(filepath.Dir(orphan), 0755)).To(Succeed())
Expect(os.WriteFile(orphan, nil, 0600)).To(Succeed())
var actual []string
err := fs.Reload(func(key, name string) { actual = append(actual, name) })
Expect(err).To(BeNil())
Expect(actual).To(BeEmpty())
_, orphanErr := os.Stat(orphan)
Expect(os.IsNotExist(orphanErr)).To(BeTrue())
2021-04-22 14:02:42 -04:00
})
})
})