fix(ioutils): recover Latin-1/Windows-1252 text in playlists and lyrics - #6037

UTF8Reader decoded non-BOM input strictly as UTF-8, so a byte that was not
valid UTF-8 became the replacement character U+FFFD. Playlists and lyrics
saved by Windows editors are commonly Windows-1252/Latin-1, where accented
characters are single high bytes (for example é as 0xE9). Those bytes were
mangled into U+FFFD, so an M3U entry like 'PokéMon' no longer matched the
scanned library path and the track was silently skipped, while pure-ASCII
entries in the same playlist imported fine.

UTF8Reader now keeps valid UTF-8 untouched and decodes any invalid byte as
Windows-1252 (a superset of Latin-1), recovering the accented characters
instead of losing them. BOM handling and UTF-16 transcoding are unchanged.

Closes #6037

Signed-off-by: Steve <stevejason639@gmail.com>

core/playlists/import_test.go

@@ -21,6 +21,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/zeebo/xxh3"
+	"golang.org/x/text/encoding/charmap"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -100,6 +101,34 @@ var _ = Describe("Playlists - Import", func() {
 				Expect(pls.Tracks[0].Path).To(Equal("tests/fixtures/playlists/test.mp3"))
 			})
 
+			It("matches accented tracks from a Windows-1252/Latin-1 encoded playlist (issue #6037)", func() {
+				tmpDir := GinkgoT().TempDir()
+
+				// The track exists in the library with a correctly UTF-8 encoded name
+				// mixing CJK folders with accented Latin characters, as reported.
+				dbPath := "PokéMon Black Và White.mp3"
+
+				// The playlist was saved by a Windows editor as Windows-1252, so the
+				// accented characters are single high bytes (é=0xE9, à=0xE0) rather
+				// than UTF-8. Encode the same name to Windows-1252 for the on-disk file.
+				latin1Line, err := charmap.Windows1252.NewEncoder().String(dbPath)
+				Expect(err).ToNot(HaveOccurred())
+				Expect([]byte(latin1Line)).To(ContainElement(byte(0xE9)))
+
+				plsFile := filepath.Join(tmpDir, "test.m3u")
+				Expect(os.WriteFile(plsFile, []byte(latin1Line+"\n"), 0600)).To(Succeed())
+
+				mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}})
+				ds.MockedMediaFile = &mockedMediaFileFromListRepo{data: []string{dbPath}}
+				ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds))
+
+				plsFolder := &model.Folder{ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: ""}
+				pls, err := ps.ImportFromFolder(ctx, plsFolder, "test.m3u")
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pls.Tracks).To(HaveLen(1))
+				Expect(pls.Tracks[0].Path).To(Equal(dbPath))
+			})
+
 			It("parses #EXTALBUMARTURL with HTTP URL", func() {
 				conf.Server.EnableM3UExternalAlbumArt = true
 

utils/ioutils/ioutils.go

@@ -3,24 +3,83 @@ package ioutils
 import (
 	"io"
 	"os"
+	"unicode/utf8"
 
+	"golang.org/x/text/encoding/charmap"
 	"golang.org/x/text/encoding/unicode"
 	"golang.org/x/text/transform"
 )
 
-// UTF8Reader wraps an io.Reader to handle Byte Order Mark (BOM) properly.
-// It strips UTF-8 BOM if present, and converts UTF-16 (LE/BE) to UTF-8.
-// This is particularly useful for reading user-provided text files (like LRC lyrics,
-// playlists) that may have been created on Windows, which often adds BOM markers.
+// UTF8Reader wraps an io.Reader so downstream code always sees UTF-8, whatever
+// encoding a user-provided text file (LRC lyrics, M3U playlists) happened to be
+// saved in. It honors a Byte Order Mark when present: a UTF-8 BOM is stripped,
+// and UTF-16 (LE/BE) is transcoded to UTF-8. When there is no BOM the bytes are
+// read as UTF-8, and any byte that is not valid UTF-8 is decoded as Windows-1252
+// (a superset of Latin-1). That recovers accented characters from the legacy
+// single-byte encodings Windows editors still emit, instead of replacing them
+// with U+FFFD and losing the match. Valid UTF-8 always passes through untouched.
 //
 // Reference: https://en.wikipedia.org/wiki/Byte_order_mark
 func UTF8Reader(r io.Reader) io.Reader {
-	return transform.NewReader(r, unicode.BOMOverride(unicode.UTF8.NewDecoder()))
+	return transform.NewReader(r, unicode.BOMOverride(utf8OrWindows1252{}))
 }
 
-// UTF8ReadFile reads the named file and returns its contents as a byte slice,
-// automatically handling BOM markers. It's similar to os.ReadFile but strips
-// UTF-8 BOM and converts UTF-16 encoded files to UTF-8.
+// utf8OrWindows1252 passes valid UTF-8 through unchanged and decodes any invalid
+// byte as Windows-1252. UTF-8 is preferred, so well-formed input is never
+// altered; the fallback only rescues bytes that could not be valid UTF-8, which
+// is the hallmark of a legacy Latin-1/Windows-1252 file.
+type utf8OrWindows1252 struct{}
+
+func (utf8OrWindows1252) Reset() {}
+
+func (utf8OrWindows1252) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for nSrc < len(src) {
+		b := src[nSrc]
+
+		// ASCII fast path: identical in UTF-8 and Windows-1252.
+		if b < utf8.RuneSelf {
+			if nDst >= len(dst) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			dst[nDst] = b
+			nDst++
+			nSrc++
+			continue
+		}
+
+		// A byte >= 0x80 begins a multi-byte UTF-8 sequence. If the remaining
+		// input might still hold an incomplete sequence, ask for more before
+		// judging it invalid, so a rune split across reads is not misdecoded.
+		if !atEOF && !utf8.FullRune(src[nSrc:]) {
+			return nDst, nSrc, transform.ErrShortSrc
+		}
+
+		if r, size := utf8.DecodeRune(src[nSrc:]); r != utf8.RuneError || size > 1 {
+			// Valid UTF-8 rune (including a genuine U+FFFD): copy it verbatim.
+			if nDst+size > len(dst) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			copy(dst[nDst:], src[nSrc:nSrc+size])
+			nDst += size
+			nSrc += size
+			continue
+		}
+
+		// Not valid UTF-8: decode this single byte as Windows-1252.
+		r := charmap.Windows1252.DecodeByte(b)
+		if nDst+utf8.RuneLen(r) > len(dst) {
+			return nDst, nSrc, transform.ErrShortDst
+		}
+		nDst += utf8.EncodeRune(dst[nDst:], r)
+		nSrc++
+	}
+	return nDst, nSrc, nil
+}
+
+// UTF8ReadFile reads the named file and returns its contents as UTF-8 bytes.
+// It's like os.ReadFile but runs the data through UTF8Reader, so BOMs are
+// stripped, UTF-16 is transcoded, and legacy Windows-1252/Latin-1 bytes are
+// recovered rather than replaced with U+FFFD.
 func UTF8ReadFile(filename string) ([]byte, error) {
 	file, err := os.Open(filename)
 	if err != nil {

utils/ioutils/ioutils_test.go

@@ -81,6 +81,49 @@ var _ = Describe("UTF8Reader", func() {
 			Expect(string(output)).To(Equal(""))
 		})
 	})
+
+	Context("when reading Windows-1252/Latin-1 encoded text (issue #6037)", func() {
+		It("decodes accented bytes instead of emitting U+FFFD", func() {
+			// "PokéMon" and "Và" with é (0xE9) and à (0xE0) as single Latin-1 bytes.
+			input := []byte{'P', 'o', 'k', 0xE9, 'M', 'o', 'n', ' ', 'V', 0xE0}
+			reader := UTF8Reader(bytes.NewReader(input))
+
+			output, err := io.ReadAll(reader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(string(output)).To(Equal("PokéMon Và"))
+		})
+
+		It("decodes Windows-1252 specific bytes in the 0x80-0x9F range", func() {
+			// 0x80 is the Euro sign and 0x93/0x94 are curly quotes in Windows-1252,
+			// none of which exist in plain Latin-1.
+			input := []byte{0x80, '5', ' ', 0x93, 'h', 'i', 0x94}
+			reader := UTF8Reader(bytes.NewReader(input))
+
+			output, err := io.ReadAll(reader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(string(output)).To(Equal("€5 “hi”"))
+		})
+
+		It("leaves valid multi-byte UTF-8 untouched", func() {
+			// A path that mixes CJK and accented Latin, already valid UTF-8, must
+			// pass through byte-for-byte so real UTF-8 files are never corrupted.
+			input := []byte("收藏/PokéMon Và White.mp3")
+			reader := UTF8Reader(bytes.NewReader(input))
+
+			output, err := io.ReadAll(reader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(output).To(Equal(input))
+		})
+
+		It("preserves a genuine U+FFFD present in valid UTF-8", func() {
+			input := []byte("a<>b")
+			reader := UTF8Reader(bytes.NewReader(input))
+
+			output, err := io.ReadAll(reader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(output).To(Equal(input))
+		})
+	})
 })
 
 var _ = Describe("UTF8ReadFile", func() {