From ba16ab41581ac5d9896695bd45dbe715ff2f8450 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 4 Sep 2026 23:23:50 -0400 Subject: [PATCH] perf(pglite): artist stats as hash joins, one statement for a full refresh; bigint library size The stats UPDATE now joins the materialized counters (with a LEFT JOIN so artists without files still get '{}') instead of a correlated subquery, and a full refresh runs as a single statement: 3.3 s for 30k artists locally versus 30 batches. Batches of 1000 remain for partial refreshes (0.24 s each). library.total_size was integer; a 2 TB library overflowed it at the end of the first scan. It is bigint now, matching the Go int64. The bridge no longer synthesizes a reply for Terminate, which never gets one by design. --- .../20260905004449_initial_schema.sql | 2 +- db/pglite/pglite.go | 2 +- persistence/artist_repository.go | 71 +++++++++---------- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/db/migrations/20260905004449_initial_schema.sql b/db/migrations/20260905004449_initial_schema.sql index d473551d2..e146b3002 100644 --- a/db/migrations/20260905004449_initial_schema.sql +++ b/db/migrations/20260905004449_initial_schema.sql @@ -161,7 +161,7 @@ CREATE TABLE IF NOT EXISTS library ( last_scan_at timestamp not null default '0001-01-01 00:00:00', updated_at timestamp not null default current_timestamp, created_at timestamp not null default current_timestamp - , last_scan_started_at timestamp default '0001-01-01 00:00:00' not null, full_scan_in_progress boolean default false not null, total_songs integer default 0 not null, total_albums integer default 0 not null, total_artists integer default 0 not null, total_folders integer default 0 not null, total_files integer default 0 not null, total_missing_files integer default 0 not null, total_size integer default 0 not null, total_duration double precision DEFAULT 0, default_new_users boolean DEFAULT false); + , last_scan_started_at timestamp default '0001-01-01 00:00:00' not null, full_scan_in_progress boolean default false not null, total_songs integer default 0 not null, total_albums integer default 0 not null, total_artists integer default 0 not null, total_folders integer default 0 not null, total_files integer default 0 not null, total_missing_files integer default 0 not null, total_size bigint default 0 not null, total_duration double precision DEFAULT 0, default_new_users boolean DEFAULT false); CREATE TABLE IF NOT EXISTS "library_artist" ( library_id integer NOT NULL DEFAULT 1 diff --git a/db/pglite/pglite.go b/db/pglite/pglite.go index 432252ce3..9868dd6c6 100644 --- a/db/pglite/pglite.go +++ b/db/pglite/pglite.go @@ -467,7 +467,7 @@ func (pg *PGlite) handleConn(conn net.Conn, ioBase string) { pg.wasmMu.Unlock() if !handshakeDone { replies, handshakeDone = pg.fixHandshake(replies) - } else { + } else if packet[0] != 'X' { // Terminate gets no reply by design fixed := ensureReadyForQuery(replies, trapErr) if len(fixed) != len(replies) { fmt.Fprintf(pg.cfg.Stderr, "# bridge: incomplete reply to %s (trap=%v); synthesized error+ReadyForQuery\n", wireTags(packet), trapErr != nil) diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index acc163fee..a96636f17 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -496,7 +496,7 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) { sum(mf.size) AS size FROM media_file_artists mfa JOIN media_file mf ON mfa.media_file_id = mf.id - WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders + WHERE mfa.artist_id ARTIST_FILTER GROUP BY mfa.artist_id, mf.library_id, mfa.role ), artist_total_counters AS MATERIALIZED ( @@ -508,7 +508,7 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) { sum(mf.size) AS size FROM media_file_artists mfa JOIN media_file mf ON mfa.media_file_id = mf.id - WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders + WHERE mfa.artist_id ARTIST_FILTER GROUP BY mfa.artist_id, mf.library_id ), artist_participant_counter AS MATERIALIZED ( @@ -520,7 +520,7 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) { sum(mf.size) AS size FROM media_file_artists mfa JOIN media_file mf ON mfa.media_file_id = mf.id - WHERE mfa.artist_id IN (ROLE_IDS_PLACEHOLDER) -- Will replace with actual placeholders + WHERE mfa.artist_id ARTIST_FILTER AND mfa.role IN ('albumartist', 'artist') GROUP BY mfa.artist_id, mf.library_id ), @@ -542,46 +542,41 @@ func (r *artistRepository) RefreshStats(allArtists bool) (int64, error) { GROUP BY artist_id, library_id ) UPDATE library_artist - SET stats = coalesce((SELECT counters FROM library_artist_counters lac - WHERE lac.artist_id = library_artist.artist_id - AND lac.library_id = library_artist.library_id), '{}'::jsonb) - WHERE library_artist.artist_id IN (ROLE_IDS_PLACEHOLDER);` // Will replace with actual placeholders + SET stats = coalesce(x.counters, '{}'::jsonb) + FROM (SELECT la.artist_id, la.library_id, lac.counters + FROM library_artist la + LEFT JOIN library_artist_counters lac + ON lac.artist_id = la.artist_id AND lac.library_id = la.library_id + WHERE la.artist_id ARTIST_FILTER) x + WHERE x.artist_id = library_artist.artist_id AND x.library_id = library_artist.library_id;` // Will replace with actual placeholders var totalRowsAffected int64 = 0 - const batchSize = 1000 - - batchCounter := 0 - for artistIDBatch := range slice.CollectChunks(slices.Values(allTouchedArtistIDs), batchSize) { - batchCounter++ - log.Trace(r.ctx, "RefreshStats: Processing batch", "batchNum", batchCounter, "batchSize", len(artistIDBatch)) - - // Create placeholders for each ID in the IN clauses - placeholders := make([]string, len(artistIDBatch)) - for i := range artistIDBatch { - placeholders[i] = "?" - } - // Don't add extra parentheses, the IN clause already expects them in SQL syntax - inClause := strings.Join(placeholders, ",") - - // Replace the placeholder markers with actual SQL placeholders - batchSQL := strings.Replace(batchUpdateStatsSQL, "ROLE_IDS_PLACEHOLDER", inClause, 4) - - // Create a single parameter array with all IDs (repeated 4 times for each IN clause) - // We need to repeat each ID 4 times (once for each IN clause) - args := make([]any, 4*len(artistIDBatch)) - for idx, id := range artistIDBatch { - for i := range 4 { - startIdx := i * len(artistIDBatch) - args[startIdx+idx] = id + // A full refresh runs as one statement: the materialized CTEs make it a few hash joins + // (3 s for 30k artists locally), while 1000-artist batches repeat the media_file scans. + batches := [][]string{allTouchedArtistIDs} + if !allArtists { + batches = slices.Collect(slice.CollectChunks(slices.Values(allTouchedArtistIDs), 1000)) + } + for batchNum, batch := range batches { + filter, args := "<> ''", []any(nil) + if !allArtists { + placeholders := make([]string, len(batch)) + for i := range batch { + placeholders[i] = "?" + } + filter = "IN (" + strings.Join(placeholders, ",") + ")" + // the filter appears 4 times, so the ids are bound 4 times + for range 4 { + for _, id := range batch { + args = append(args, id) + } } } - - // Now use Expr with the expanded SQL and all parameters - sqlizer := Expr(batchSQL, args...) - - rowsAffected, err := r.executeSQL(sqlizer) + log.Trace(r.ctx, "RefreshStats: Processing batch", "batchNum", batchNum+1, "batchSize", len(batch)) + batchSQL := strings.ReplaceAll(batchUpdateStatsSQL, "ARTIST_FILTER", filter) + rowsAffected, err := r.executeSQL(Expr(batchSQL, args...)) if err != nil { - return totalRowsAffected, fmt.Errorf("executing batch update for artist stats (batch %d): %w", batchCounter, err) + return totalRowsAffected, fmt.Errorf("executing batch update for artist stats (batch %d): %w", batchNum+1, err) } totalRowsAffected += rowsAffected }