fix(ioutils): recover Latin-1/Windows-1252 text in playlists and lyrics - #6037

UTF8Reader decoded non-BOM input strictly as UTF-8, so a byte that was not
valid UTF-8 became the replacement character U+FFFD. Playlists and lyrics
saved by Windows editors are commonly Windows-1252/Latin-1, where accented
characters are single high bytes (for example é as 0xE9). Those bytes were
mangled into U+FFFD, so an M3U entry like 'PokéMon' no longer matched the
scanned library path and the track was silently skipped, while pure-ASCII
entries in the same playlist imported fine.

UTF8Reader now keeps valid UTF-8 untouched and decodes any invalid byte as
Windows-1252 (a superset of Latin-1), recovering the accented characters
instead of losing them. BOM handling and UTF-16 transcoding are unchanged.

Closes #6037

Signed-off-by: Steve <stevejason639@gmail.com>
This commit is contained in:
Steve 2026-09-01 12:48:56 +05:30
commit da0899de2b
3 changed files with 139 additions and 8 deletions

View file

@ -21,6 +21,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/zeebo/xxh3"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/unicode/norm"
)
@ -100,6 +101,34 @@ var _ = Describe("Playlists - Import", func() {
Expect(pls.Tracks[0].Path).To(Equal("tests/fixtures/playlists/test.mp3"))
})
It("matches accented tracks from a Windows-1252/Latin-1 encoded playlist (issue #6037)", func() {
tmpDir := GinkgoT().TempDir()
// The track exists in the library with a correctly UTF-8 encoded name
// mixing CJK folders with accented Latin characters, as reported.
dbPath := "PokéMon Black Và White.mp3"
// The playlist was saved by a Windows editor as Windows-1252, so the
// accented characters are single high bytes (é=0xE9, à=0xE0) rather
// than UTF-8. Encode the same name to Windows-1252 for the on-disk file.
latin1Line, err := charmap.Windows1252.NewEncoder().String(dbPath)
Expect(err).ToNot(HaveOccurred())
Expect([]byte(latin1Line)).To(ContainElement(byte(0xE9)))
plsFile := filepath.Join(tmpDir, "test.m3u")
Expect(os.WriteFile(plsFile, []byte(latin1Line+"\n"), 0600)).To(Succeed())
mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}})
ds.MockedMediaFile = &mockedMediaFileFromListRepo{data: []string{dbPath}}
ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds))
plsFolder := &model.Folder{ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: ""}
pls, err := ps.ImportFromFolder(ctx, plsFolder, "test.m3u")
Expect(err).ToNot(HaveOccurred())
Expect(pls.Tracks).To(HaveLen(1))
Expect(pls.Tracks[0].Path).To(Equal(dbPath))
})
It("parses #EXTALBUMARTURL with HTTP URL", func() {
conf.Server.EnableM3UExternalAlbumArt = true