package scanner import ( "context" "fmt" "maps" "slices" "sync/atomic" "time" ppl "github.com/google/go-pipeline/pkg/pipeline" "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/core/playlists" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/run" "github.com/navidrome/navidrome/utils/slice" ) type scannerImpl struct { ds model.DataStore pls playlists.Playlists } // scanState holds the state of an in-progress scan, to be passed to the various phases type scanState struct { progress chan<- *ProgressInfo fullScan bool changesDetected atomic.Bool libraries model.Libraries // Store libraries list for consistency across phases targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans totalLibraryCount int // Total number of libraries (unfiltered), for cross-library move detection failedLibs map[int]bool // Libraries that could not be scanned in this run } func (s *scanState) sendProgress(info *ProgressInfo) { if s.progress != nil { s.progress <- info } } func (s *scanState) isSelectiveScan() bool { return len(s.targets) > 0 } func (s *scanState) sendWarning(msg string) { s.sendProgress(&ProgressInfo{Warning: msg}) } func (s *scanState) markFailed(libID int) { if s.failedLibs == nil { s.failedLibs = map[int]bool{} } s.failedLibs[libID] = true } func (s *scanState) sendError(err error) { s.sendProgress(&ProgressInfo{Error: err.Error()}) } func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) { startTime := time.Now() state := scanState{ progress: progress, fullScan: fullScan, changesDetected: atomic.Bool{}, } // Set changesDetected to true for full scans to ensure all maintenance operations run if fullScan { state.changesDetected.Store(true) } // Get libraries and optionally filter by targets allLibs, err := s.ds.Library().GetAll(ctx) if err != nil { state.sendWarning(fmt.Sprintf("getting libraries: %s", err)) return } state.totalLibraryCount = len(allLibs) if len(targets) > 0 { // Selective scan: filter libraries and build targets map state.targets = make(map[int][]string) libPaths := slice.ToMap(allLibs, func(lib model.Library) (int, string) { return lib.ID, lib.Path }) for _, target := range targets { folderPath := model.LibraryRelativePath(libPaths[target.LibraryID], target.FolderPath) if folderPath == "" { folderPath = "." } state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath) } // Filter libraries to only those in targets state.libraries = slice.Filter(allLibs, func(lib model.Library) bool { return len(state.targets[lib.ID]) > 0 }) log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries), "numTargets", len(targets)) } else { // Full library scan state.libraries = allLibs log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries)) } // Store scan type and start time scanType := "quick" if state.fullScan { scanType = "full" } if state.isSelectiveScan() { scanType += "-selective" } _ = s.ds.Property().Put(ctx, consts.LastScanTypeKey, scanType) _ = s.ds.Property().Put(ctx, consts.LastScanStartTimeKey, startTime.Format(time.RFC3339)) // if there was a full scan in progress, force a full scan if !state.fullScan { for _, lib := range state.libraries { // A pending PID rescan already restarts in full through its own job if lib.NeedsPIDRescan() { continue } if lib.FullScanInProgress { log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name) state.fullScan = true if state.isSelectiveScan() { _ = s.ds.Property().Put(ctx, consts.LastScanTypeKey, "full-selective") } else { _ = s.ds.Property().Put(ctx, consts.LastScanTypeKey, "full") } break } } } // Prepare libraries for scanning (initialize LastScanStartedAt if needed) err = s.prepareLibrariesForScan(ctx, &state) if err != nil { log.Error(ctx, "Scanner: Error preparing libraries for scan", err) state.sendError(err) return } err = run.Sequentially( // Phase 1: Scan all libraries and import new/updated files runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds)), // Phase 2: Process missing files, checking for moves runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)), // Phases 3 and 4 can be run in parallel run.Parallel( // Phase 3: Refresh all new/changed albums and update artists runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds)), // Phase 4: Import/update playlists runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls)), ), // Final Steps (cannot be parallelized): // Run GC if there were any changes (Remove dangling tracks, empty albums and artists, and orphan annotations) s.runGC(ctx, &state), // Queue artwork for entities that never resolved (after GC, so nothing dangling is queued) s.runEnqueueMissingArtwork(ctx, &state), // Refresh artist and tags stats s.runRefreshStats(ctx, &state), // Update last_scan_completed_at for all libraries s.runUpdateLibraries(ctx, &state), ) if err != nil { log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err) _ = s.ds.Property().Put(ctx, consts.LastScanErrorKey, err.Error()) state.sendError(err) return } _ = s.ds.Property().Put(ctx, consts.LastScanErrorKey, "") if state.changesDetected.Load() { state.sendProgress(&ProgressInfo{ChangesDetected: true}) } if state.isSelectiveScan() { log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets)) } else { log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime)) } } // prepareLibrariesForScan initializes the scan for all libraries in the state. // It calls ScanBegin for libraries that haven't started scanning yet (LastScanStartedAt is zero), // reloads them to get the updated state, and filters out any libraries that fail to initialize. func (s *scannerImpl) prepareLibrariesForScan(ctx context.Context, state *scanState) error { var successfulLibs []model.Library for _, lib := range state.libraries { // A library with a changed PID config restarts its scan: resuming would skip the folders that // the interrupted scan already processed with the old config pidRescan := lib.NeedsPIDRescan() if lib.LastScanStartedAt.IsZero() || pidRescan { // This is a new scan - mark it as started err := s.ds.WithTxRetry(ctx, func(ctx context.Context, tx model.DataStore) error { return tx.Library().ScanBegin(ctx, lib.ID, state.fullScan || pidRescan) }, "scanner: begin library scan") if err != nil { log.Error(ctx, "Scanner: Error marking scan start", "lib", lib.Name, err) state.sendWarning(err.Error()) continue } // Reload library to get updated state (timestamps, etc.) reloadedLib, err := s.ds.Library().Get(ctx, lib.ID) if err != nil { log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err) state.sendWarning(err.Error()) continue } lib = *reloadedLib } else { // This is a resumed scan log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress) } successfulLibs = append(successfulLibs, lib) } if len(successfulLibs) == 0 { return fmt.Errorf("no libraries available for scanning") } // Update state with only successfully initialized libraries state.libraries = successfulLibs return nil } func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error { return func() error { state.sendProgress(&ProgressInfo{ForceUpdate: true}) return s.ds.WithTxRetry(ctx, func(ctx context.Context, tx model.DataStore) error { if state.changesDetected.Load() { start := time.Now() // For selective scans, extract library IDs to scope GC operations var libraryIDs []int if state.isSelectiveScan() { libraryIDs = slices.Collect(maps.Keys(state.targets)) log.Debug(ctx, "Scanner: Running selective GC", "libraryIDs", libraryIDs) } if err := tx.GC(ctx, libraryIDs...); err != nil { return fmt.Errorf("running GC: %w", err) } log.Debug(ctx, "Scanner: GC completed", "elapsed", time.Since(start)) } else { log.Debug(ctx, "Scanner: No changes detected, skipping GC") } return nil }, "scanner: GC") } } // runEnqueueMissingArtwork is the safety net for entities phase 1 never enqueued. func (s *scannerImpl) runEnqueueMissingArtwork(ctx context.Context, state *scanState) func() error { return func() error { if !state.changesDetected.Load() { log.Debug(ctx, "Scanner: No changes detected, skipping artwork enqueue") return nil } start := time.Now() var total int64 for _, kind := range []model.Kind{model.KindAlbumArtwork, model.KindArtistArtwork} { var n int64 err := s.ds.WithTxRetry(ctx, func(ctx context.Context, tx model.DataStore) error { var err error n, err = tx.ArtworkQueue().EnqueueAllMissing(ctx, kind, model.ArtworkPriorityScan) return err }, "scanner: enqueue missing artwork") if err != nil { log.Error(ctx, "Scanner: Error enqueueing missing artwork", "kind", kind, err) return fmt.Errorf("enqueueing missing artwork: %w", err) } total += n } log.Debug(ctx, "Scanner: Enqueued missing artwork", "items", total, "elapsed", time.Since(start)) return nil } } func (s *scannerImpl) runRefreshStats(ctx context.Context, state *scanState) func() error { return func() error { if !state.changesDetected.Load() { log.Debug(ctx, "Scanner: No changes detected, skipping refreshing stats") return nil } start := time.Now() stats, err := s.ds.Artist().RefreshStats(ctx, state.fullScan) if err != nil { log.Error(ctx, "Scanner: Error refreshing artists stats", err) return fmt.Errorf("refreshing artists stats: %w", err) } log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start)) start = time.Now() err = s.ds.WithTxRetry(ctx, func(ctx context.Context, tx model.DataStore) error { return tx.Tag().UpdateCounts(ctx) }, "scanner: update tag counts") if err != nil { log.Error(ctx, "Scanner: Error updating tag counts", err) return fmt.Errorf("updating tag counts: %w", err) } log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start)) return nil } } func (s *scannerImpl) runUpdateLibraries(ctx context.Context, state *scanState) func() error { return func() error { start := time.Now() return s.ds.WithTxRetry(ctx, func(ctx context.Context, tx model.DataStore) error { for _, lib := range state.libraries { if err := tx.Library().ScanEnd(ctx, lib.ID); err != nil { return fmt.Errorf("updating last scan completed for %s: %w", lib.Name, err) } // A selective scan covers only part of the library, so the rest may still use the old PID // config. A library that could not be scanned did not apply it either. if !state.isSelectiveScan() && !state.failedLibs[lib.ID] { if err := tx.Library().SetScannedPID(ctx, lib.ID, lib.EffectivePID()); err != nil { return fmt.Errorf("updating PID conf for %s: %w", lib.Name, err) } } if state.changesDetected.Load() { log.Debug(ctx, "Scanner: Refreshing library stats", "lib", lib.Name) if err := tx.Library().RefreshStats(ctx, lib.ID); err != nil { return fmt.Errorf("refreshing library stats for %s: %w", lib.Name, err) } } else { log.Debug(ctx, "Scanner: No changes detected, skipping library stats refresh", "lib", lib.Name) } } log.Debug(ctx, "Scanner: Updated libraries after scan", "elapsed", time.Since(start), "numLibraries", len(state.libraries)) return nil }, "scanner: update libraries") } } type phase[T any] interface { producer() ppl.Producer[T] stages() []ppl.Stage[T] finalize(error) error description() string } func runPhase[T any](ctx context.Context, phaseNum int, phase phase[T]) func() error { return func() error { log.Debug(ctx, fmt.Sprintf("Scanner: Starting phase %d: %s", phaseNum, phase.description())) start := time.Now() producer := phase.producer() stages := phase.stages() // Prepend a counter stage to the phase's pipeline counter, countStageFn := countTasks[T]() stages = append([]ppl.Stage[T]{ppl.NewStage(countStageFn, ppl.Name("count tasks"))}, stages...) var err error if log.IsGreaterOrEqualTo(log.LevelDebug) { var m *ppl.Metrics m, err = ppl.Measure(producer, stages...) log.Info(ctx, "Scanner: "+m.String(), err) } else { err = ppl.Do(producer, stages...) } err = phase.finalize(err) if err != nil { log.Error(ctx, fmt.Sprintf("Scanner: Error processing libraries in phase %d", phaseNum), "elapsed", time.Since(start), err) } else { log.Debug(ctx, fmt.Sprintf("Scanner: Finished phase %d", phaseNum), "elapsed", time.Since(start), "totalTasks", counter.Load()) } return err } } func countTasks[T any]() (*atomic.Int64, func(T) (T, error)) { counter := atomic.Int64{} return &counter, func(in T) (T, error) { counter.Add(1) return in, nil } } var _ scanner = (*scannerImpl)(nil)