remote/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2026-10-11 03:47:18 +02:00
Code Issues Projects Releases Packages Wiki Activity

perf(pglite): artist stats as hash joins, one statement for a full refresh; bigint library size

Browse source 
The stats UPDATE now joins the materialized counters (with a LEFT JOIN
so artists without files still get '{}') instead of a correlated
subquery, and a full refresh runs as a single statement: 3.3 s for 30k
artists locally versus 30 batches. Batches of 1000 remain for partial
refreshes (0.24 s each).

library.total_size was integer; a 2 TB library overflowed it at the end
of the first scan. It is bigint now, matching the Go int64.

The bridge no longer synthesizes a reply for Terminate, which never gets
one by design.
This commit is contained in:
Deluan 2026-09-04 23:23:50 -04:00
commit ba16ab4158
3 changed files with 35 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

2
db/migrations/20260905004449_initial_schema.sql
View file

@ -161,7 +161,7 @@ CREATE TABLE IF NOT EXISTS library (
last_scan_at timestamp not null default '0001-01-01 00:00:00',
updated_at timestamp not null default current_timestamp,
created_at timestamp not null default current_timestamp
, last_scan_started_at timestamp default '0001-01-01 00:00:00' not null, full_scan_in_progress boolean default false not null, total_songs integer default 0 not null, total_albums integer default 0 not null, total_artists integer default 0 not null, total_folders integer default 0 not null, total_files integer default 0 not null, total_missing_files integer default 0 not null, total_size integer default 0 not null, total_duration double precision DEFAULT 0, default_new_users boolean DEFAULT false);
, last_scan_started_at timestamp default '0001-01-01 00:00:00' not null, full_scan_in_progress boolean default false not null, total_songs integer default 0 not null, total_albums integer default 0 not null, total_artists integer default 0 not null, total_folders integer default 0 not null, total_files integer default 0 not null, total_missing_files integer default 0 not null, total_size bigint default 0 not null, total_duration double precision DEFAULT 0, default_new_users boolean DEFAULT false);
CREATE TABLE IF NOT EXISTS "library_artist"
(
library_id integer NOT NULL DEFAULT 1

2
db/pglite/pglite.go
View file

@ -467,7 +467,7 @@ func (pg *PGlite) handleConn(conn net.Conn, ioBase string) {
pg.wasmMu.Unlock()
if !handshakeDone {
replies, handshakeDone = pg.fixHandshake(replies)
} else {
} else if packet[0] != 'X' { // Terminate gets no reply by design
fixed := ensureReadyForQuery(replies, trapErr)
if len(fixed) != len(replies) {
fmt.Fprintf(pg.cfg.Stderr, "# bridge: incomplete reply to %s (trap=%v); synthesized error+ReadyForQuery\n", wireTags(packet), trapErr != nil)

71
persistence/artist_repository.go
View file

@ -496,7 +496,7 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) {
sum(mf.size) AS size
FROM media_file_artists mfa
JOIN media_file mf ON mfa.media_file_id = mf.id
WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders
WHERE mfa.artist_id ARTIST_FILTER
GROUP BY mfa.artist_id, mf.library_id, mfa.role
),
artist_total_counters AS MATERIALIZED (
@ -508,7 +508,7 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) {
sum(mf.size) AS size
FROM media_file_artists mfa
JOIN media_file mf ON mfa.media_file_id = mf.id
WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders
WHERE mfa.artist_id ARTIST_FILTER
GROUP BY mfa.artist_id, mf.library_id
),
artist_participant_counter AS MATERIALIZED (
@ -520,7 +520,7 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) {
sum(mf.size) AS size
FROM media_file_artists mfa
JOIN media_file mf ON mfa.media_file_id = mf.id
WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders
WHERE mfa.artist_id ARTIST_FILTER
AND mfa.role IN ('albumartist', 'artist')
GROUP BY mfa.artist_id, mf.library_id
),
@ -542,46 +542,41 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) {
GROUP BY artist_id, library_id
)
UPDATE library_artist
SET stats = coalesce((SELECT counters FROM library_artist_counters lac
WHERE lac.artist_id = library_artist.artist_id
AND lac.library_id = library_artist.library_id), '{}'::jsonb)
WHERE library_artist.artist_id IN (ROLE_IDS_PLACEHOLDER);` // Will replace with actual placeholders
SET stats = coalesce(x.counters, '{}'::jsonb)
FROM (SELECT la.artist_id, la.library_id, lac.counters
FROM library_artist la
LEFT JOIN library_artist_counters lac
ON lac.artist_id = la.artist_id AND lac.library_id = la.library_id
WHERE la.artist_id ARTIST_FILTER) x
WHERE x.artist_id = library_artist.artist_id AND x.library_id = library_artist.library_id;` // Will replace with actual placeholders
var totalRowsAffected int64 = 0
const batchSize = 1000
batchCounter := 0
for artistIDBatch := range slice.CollectChunks(slices.Values(allTouchedArtistIDs), batchSize) {
batchCounter++
log.Trace(r.ctx, "RefreshStats: Processing batch", "batchNum", batchCounter, "batchSize", len(artistIDBatch))
// Create placeholders for each ID in the IN clauses
placeholders := make([]string, len(artistIDBatch))
for i := range artistIDBatch {
placeholders[i] = "?"
}
// Don't add extra parentheses, the IN clause already expects them in SQL syntax
inClause := strings.Join(placeholders, ",")
// Replace the placeholder markers with actual SQL placeholders
batchSQL := strings.Replace(batchUpdateStatsSQL, "ROLE_IDS_PLACEHOLDER", inClause, 4)
// Create a single parameter array with all IDs (repeated 4 times for each IN clause)
// We need to repeat each ID 4 times (once for each IN clause)
args := make([]any, 4*len(artistIDBatch))
for idx, id := range artistIDBatch {
for i := range 4 {
startIdx := i * len(artistIDBatch)
args[startIdx+idx] = id
// A full refresh runs as one statement: the materialized CTEs make it a few hash joins
// (3 s for 30k artists locally), while 1000-artist batches repeat the media_file scans.
batches := [][]string{allTouchedArtistIDs}
if !allArtists {
batches = slices.Collect(slice.CollectChunks(slices.Values(allTouchedArtistIDs), 1000))
}
for batchNum, batch := range batches {
filter, args := "<> ''", []any(nil)
if !allArtists {
placeholders := make([]string, len(batch))
for i := range batch {
placeholders[i] = "?"
}
filter = "IN (" + strings.Join(placeholders, ",") + ")"
// the filter appears 4 times, so the ids are bound 4 times
for range 4 {
for _, id := range batch {
args = append(args, id)
}
}
}
// Now use Expr with the expanded SQL and all parameters
sqlizer := Expr(batchSQL, args...)
rowsAffected, err := r.executeSQL(sqlizer)
log.Trace(r.ctx, "RefreshStats: Processing batch", "batchNum", batchNum+1, "batchSize", len(batch))
batchSQL := strings.ReplaceAll(batchUpdateStatsSQL, "ARTIST_FILTER", filter)
rowsAffected, err := r.executeSQL(Expr(batchSQL, args...))
if err != nil {
return totalRowsAffected, fmt.Errorf("executing batch update for artist stats (batch %d): %w", batchCounter, err)
return totalRowsAffected, fmt.Errorf("executing batch update for artist stats (batch %d): %w", batchNum+1, err)
}
totalRowsAffected += rowsAffected
}