navidrome/persistence/sql_search_like_test.go
Deluan 7bb2c0968f perf(search): speed up CJK search with NOCASE covering indexes
Queries containing CJK (and punctuation-only) text bypass FTS5 and fall back to
LIKE-based search, because the unicode61 tokenizer indexes whole space-free CJK
runs as single tokens and cannot substring-match them. That fallback had no usable
index, so each query did a full scan of the wide media_file table across four
columns — roughly 4 seconds on a one-million-track library.

Add COLLATE NOCASE covering indexes on exactly the columns the LIKE fallback
searches (likeSearchColumns), so SQLite scans the narrow per-column indexes instead
of the full table. A plain LIKE uses them because the server runs with
case_sensitive_like = OFF. Results are unchanged; only latency improves
(~4s to ~0.3s end-to-end). The FTS path for Latin/numeric queries is untouched.

A new test asserts every likeSearchColumns column has a matching NOCASE index in
the migrated schema, so the column set and the migration cannot silently drift.
2026-06-14 23:59:23 -04:00

170 lines
5.9 KiB
Go

package persistence
import (
"context"
"fmt"
"regexp"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("newLegacySearch", func() {
It("returns non-nil for single-character query", func() {
strategy := newLegacySearch("media_file", "a")
Expect(strategy).ToNot(BeNil(), "single-char queries must not be rejected; min-length is enforced in doSearch, not here")
sql, _, err := strategy.ToSql()
Expect(err).ToNot(HaveOccurred())
Expect(sql).To(ContainSubstring("LIKE"))
})
})
var _ = Describe("legacySearchExpr", func() {
It("returns nil for empty query", func() {
Expect(legacySearchExpr("media_file", "")).To(BeNil())
})
It("generates LIKE filter for single word", func() {
expr := legacySearchExpr("media_file", "beatles")
sql, args, err := expr.ToSql()
Expect(err).ToNot(HaveOccurred())
Expect(sql).To(ContainSubstring("media_file.full_text LIKE"))
Expect(args).To(ContainElement("% beatles%"))
})
It("generates AND of LIKE filters for multiple words", func() {
expr := legacySearchExpr("media_file", "abbey road")
sql, args, err := expr.ToSql()
Expect(err).ToNot(HaveOccurred())
Expect(sql).To(ContainSubstring("AND"))
Expect(args).To(HaveLen(2))
})
})
var _ = Describe("likeSearchExpr", func() {
It("returns nil for empty query", func() {
Expect(likeSearchExpr("media_file", "")).To(BeNil())
})
It("returns nil for whitespace-only query", func() {
Expect(likeSearchExpr("media_file", " ")).To(BeNil())
})
It("generates LIKE filters against core columns for single CJK word", func() {
expr := likeSearchExpr("media_file", "周杰伦")
sql, args, err := expr.ToSql()
Expect(err).ToNot(HaveOccurred())
// Should have OR between columns for the single word
Expect(sql).To(ContainSubstring("OR"))
Expect(sql).To(ContainSubstring("media_file.title LIKE"))
Expect(sql).To(ContainSubstring("media_file.album LIKE"))
Expect(sql).To(ContainSubstring("media_file.artist LIKE"))
Expect(sql).To(ContainSubstring("media_file.album_artist LIKE"))
Expect(args).To(HaveLen(4))
for _, arg := range args {
Expect(arg).To(Equal("%周杰伦%"))
}
})
It("generates AND of OR groups for multi-word query", func() {
expr := likeSearchExpr("media_file", "周杰伦 greatest")
sql, args, err := expr.ToSql()
Expect(err).ToNot(HaveOccurred())
// Two groups AND'd together, each with 4 columns OR'd
Expect(sql).To(ContainSubstring("AND"))
Expect(args).To(HaveLen(8))
})
It("uses correct columns for album table", func() {
expr := likeSearchExpr("album", "周杰伦")
sql, args, err := expr.ToSql()
Expect(err).ToNot(HaveOccurred())
Expect(sql).To(ContainSubstring("album.name LIKE"))
Expect(sql).To(ContainSubstring("album.album_artist LIKE"))
Expect(args).To(HaveLen(2))
})
It("uses correct columns for artist table", func() {
expr := likeSearchExpr("artist", "周杰伦")
sql, args, err := expr.ToSql()
Expect(err).ToNot(HaveOccurred())
Expect(sql).To(ContainSubstring("artist.name LIKE"))
Expect(args).To(HaveLen(1))
})
It("returns nil for unknown table", func() {
Expect(likeSearchExpr("unknown_table", "周杰伦")).To(BeNil())
})
})
var _ = Describe("Legacy Integration Search", func() {
var mr model.MediaFileRepository
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
conf.Server.Search.Backend = "legacy"
ctx := log.NewContext(context.TODO())
ctx = request.WithUser(ctx, adminUser)
conn := GetDBXBuilder()
mr = NewMediaFileRepository(ctx, conn)
})
It("returns results using legacy LIKE-based search", func() {
results, err := mr.Search("Radioactivity", model.QueryOptions{Max: 10})
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(1))
Expect(results[0].Title).To(Equal("Radioactivity"))
})
It("returns empty results for single-char query (doSearch min-length guard)", func() {
results, err := mr.Search("a", model.QueryOptions{Max: 10})
Expect(err).ToNot(HaveOccurred())
Expect(results).To(BeEmpty(), "doSearch should reject single-char queries")
})
It("returns results with Max=0 (regression: must not produce LIMIT 0)", func() {
results, err := mr.Search("Beatles", model.QueryOptions{Max: 0})
Expect(err).ToNot(HaveOccurred())
Expect(results).ToNot(BeEmpty(), "Max=0 should mean no limit, not LIMIT 0")
})
})
// Guards the invariant that every column searched by the LIKE fallback has a COLLATE NOCASE
// covering index (migration add_like_search_covering_indexes). Without it a CJK/punctuation
// search does a full table scan. If you add a column to likeSearchColumns without an index,
// this test fails — keep the two in sync.
var _ = Describe("likeSearchColumns covering indexes", func() {
It("has a COLLATE NOCASE index for every searched column", func() {
var indexSQLs []string
err := GetDBXBuilder().
NewQuery("SELECT sql FROM sqlite_master WHERE type='index' AND sql IS NOT NULL").
Column(&indexSQLs)
Expect(err).ToNot(HaveOccurred())
for table, columns := range likeSearchColumns {
// Match the index's target table precisely: "ON <table> (" (case-insensitive).
tableRe := regexp.MustCompile(`(?i)\bon\s+` + regexp.QuoteMeta(table) + `\s*\(`)
for _, col := range columns {
// Match e.g. "(title COLLATE NOCASE)" — column immediately followed by the
// NOCASE collation, case-insensitive (SQLite emits both "COLLATE" and "collate").
colRe := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(col) + `\s+collate\s+nocase\b`)
found := false
for _, sql := range indexSQLs {
if tableRe.MatchString(sql) && colRe.MatchString(sql) {
found = true
break
}
}
Expect(found).To(BeTrue(),
fmt.Sprintf("missing COLLATE NOCASE index for %s.%s — add it to the "+
"add_like_search_covering_indexes migration to keep LIKE search fast", table, col))
}
}
})
})