diff --git a/ui/src/audioplayer/LyricsLineRows.jsx b/ui/src/audioplayer/LyricsLineRows.jsx new file mode 100644 index 000000000..77d9fa00b --- /dev/null +++ b/ui/src/audioplayer/LyricsLineRows.jsx @@ -0,0 +1,630 @@ +import Typography from '@material-ui/core/Typography' +import clsx from 'clsx' +import React, { + memo, + useCallback, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react' +import { buildSegmentsFromLine } from './lyricsSegments' +import { + TOKEN_ACTIVE_ALPHA, + TOKEN_FUTURE_ALPHA, + TOKEN_WIPE_EDGE_PCT, + TOKEN_WIPE_SOFT_SPREAD_PCT, +} from './lyricsKaraokeConstants' +import { + buildEmphasisStyle, + isEmphasisRole, + parseColorRGB, +} from './lyricsKaraokeStyles' +import { resolveKaraokeTokenWindows } from './lyricsTimeline' + +const EMPHASIS_TONE = 0.7 +const EMPHASIS_PAINT_OVERHANG = '0.18em' + +const emphasisPaintOverhangStyle = { + marginInlineEnd: `-${EMPHASIS_PAINT_OVERHANG}`, + paddingInlineEnd: EMPHASIS_PAINT_OVERHANG, +} + +const graphemeSegmenter = + typeof Intl !== 'undefined' && typeof Intl.Segmenter === 'function' + ? new Intl.Segmenter(undefined, { granularity: 'grapheme' }) + : null + +const splitGraphemes = (value) => { + const text = String(value || '') + return graphemeSegmenter + ? Array.from(graphemeSegmenter.segment(text), ({ segment }) => segment) + : Array.from(text) +} + +const waveTextRootStyle = { + display: 'inline-block', + position: 'relative', + verticalAlign: 'baseline', +} + +const waveTextMeasureStyle = { visibility: 'hidden' } + +const waveTextOverlayStyle = { + display: 'block', + inset: 0, + position: 'absolute', + whiteSpace: 'nowrap', +} + +const renderWaveText = (text, eligible, enabled, emphasized, className) => { + if (!eligible) return text + + return ( + + ) +} + +const tokenColor = (rgb, alpha) => { + const [r, g, b] = rgb || [255, 255, 255] + return `rgba(${r}, ${g}, ${b}, ${alpha})` +} + +const toneEmphasisRGB = (rgb) => + rgb ? rgb.map((channel) => Math.round(channel * EMPHASIS_TONE)) : rgb + +const getTokenRGB = (token, rgb) => + isEmphasisRole(token) ? toneEmphasisRGB(rgb) : rgb + +const stripLayerColors = (style) => { + const result = { ...(style || {}) } + delete result.opacity + delete result.color + delete result.WebkitTextFillColor + delete result['--lyrics-active-color'] + return result +} + +const buildLineStyle = (line, style) => { + const emphasisStyle = buildEmphasisStyle(line) + return { + ...stripLayerColors(style), + ...emphasisStyle, + ...(emphasisStyle ? { filter: `brightness(${EMPHASIS_TONE})` } : {}), + } +} + +const buildStaticEmphasisStyle = (token) => { + const emphasisStyle = buildEmphasisStyle(token) + if (!emphasisStyle) return undefined + return { + ...emphasisStyle, + filter: `brightness(${EMPHASIS_TONE})`, + } +} + +const buildTokenData = (token, rgb, line) => { + const emphasized = isEmphasisRole(token) || isEmphasisRole(line) + const tonedRGB = getTokenRGB(token, rgb) + const futureColor = tokenColor(tonedRGB, TOKEN_FUTURE_ALPHA) + const gradientDoneColor = tokenColor( + tonedRGB, + 'var(--lyrics-token-active-alpha, 1)', + ) + const softColor = tokenColor( + tonedRGB, + TOKEN_FUTURE_ALPHA + (TOKEN_ACTIVE_ALPHA - TOKEN_FUTURE_ALPHA) * 0.58, + ) + const sweepRange = 100 + TOKEN_WIPE_SOFT_SPREAD_PCT + const activeStop = `calc(var(--lyrics-progress) * ${sweepRange}% - ${TOKEN_WIPE_SOFT_SPREAD_PCT}%)` + const softStop = `calc(var(--lyrics-progress) * ${sweepRange}% - ${TOKEN_WIPE_EDGE_PCT}%)` + const futureStop = `calc(var(--lyrics-progress) * ${sweepRange}%)` + const gradient = `linear-gradient(90deg, ${gradientDoneColor} 0%, ${gradientDoneColor} ${activeStop}, ${softColor} ${softStop}, ${futureColor} ${futureStop}, ${futureColor} 100%)` + + return { + style: { + '--lyrics-progress': 0, + '--lyrics-token-active-alpha': TOKEN_ACTIVE_ALPHA, + color: 'transparent', + WebkitTextFillColor: 'transparent', + backgroundImage: gradient, + backgroundSize: '100% 100%', + backgroundClip: 'text', + WebkitBackgroundClip: 'text', + ...buildEmphasisStyle(token), + ...(emphasized ? emphasisPaintOverhangStyle : {}), + }, + presentation: { + futureAlpha: TOKEN_FUTURE_ALPHA, + activeAlpha: TOKEN_ACTIVE_ALPHA, + gradient, + }, + } +} + +const isTimedWindow = (window) => + Number.isFinite(window?.start) && Number.isFinite(window?.end) + +const useStableTokenRef = (registerToken) => { + const registerTokenRef = useRef(registerToken) + const callbacksRef = useRef(new Map()) + registerTokenRef.current = registerToken + + return useCallback( + ({ key, lineIndex, window, presentation, waveEnabled }) => { + const signature = [ + lineIndex, + window?.start ?? '', + window?.end ?? '', + presentation?.futureAlpha ?? '', + presentation?.activeAlpha ?? '', + presentation?.gradient ?? '', + waveEnabled ? 1 : 0, + ].join('\u0000') + const cached = callbacksRef.current.get(key) + if (cached?.signature === signature) return cached.callback + + const descriptor = { lineIndex, window, presentation } + const callback = (node) => { + registerTokenRef.current?.(key, descriptor, node) + } + callbacksRef.current.set(key, { signature, callback }) + return callback + }, + [], + ) +} + +export const KaraokeLineRow = memo( + ({ + lineIndex, + line, + nextLineStart, + className, + style, + tokenClassName, + waveCharacterClassName, + registerToken, + renderCharacterWave = true, + rowKey = 'main', + testId, + }) => { + const getTokenRef = useStableTokenRef(registerToken) + const segments = useMemo(() => buildSegmentsFromLine(line), [line]) + const windows = useMemo( + () => resolveKaraokeTokenWindows(line, nextLineStart), + [line, nextLineStart], + ) + const tokenRGB = useMemo( + () => (style?.color ? parseColorRGB(style.color) : [255, 255, 255]), + [style?.color], + ) + const hasTimedTokens = windows.some( + (window) => window?.start != null && window?.end != null, + ) + const lineStyle = useMemo(() => buildLineStyle(line, style), [line, style]) + + return ( + + {segments.map((segment, idx) => { + if (!segment.token) + return {segment.text} + + const window = windows[segment.tokenIndex] + const timedWindow = isTimedWindow(window) ? window : null + const key = `${lineIndex}:${rowKey}:${segment.tokenIndex}:main` + const emphasized = + isEmphasisRole(segment.token) || isEmphasisRole(line) + const tokenData = buildTokenData(segment.token, tokenRGB, line) + return ( + + {renderWaveText( + segment.text, + Boolean(timedWindow), + Boolean(renderCharacterWave && timedWindow), + emphasized, + waveCharacterClassName, + )} + + ) + })} + + ) + }, +) + +KaraokeLineRow.displayName = 'KaraokeLineRow' + +const splitTextSegments = (value) => + (value || '') + .split(/(\s+)/) + .filter((text) => text.length > 0) + .map((text) => ({ + text, + token: null, + tokenIndex: -1, + isWhitespace: /^\s+$/.test(text), + })) + +const buildPronunciationParts = (line) => { + if (!line?.value) return [] + + const segments = buildSegmentsFromLine(line) + const tokenParts = segments + .filter((segment) => segment.token && segment.text.trim()) + .map((segment) => ({ + text: segment.text.trim(), + segment, + })) + + if (tokenParts.length > 0) return tokenParts + + return splitTextSegments(line.value) + .filter((segment) => !segment.isWhitespace) + .map((segment) => ({ + text: segment.text.trim(), + segment: null, + })) +} + +const canPairPronunciationSegment = (segment, hasTokenSegments) => + Boolean( + segment.token || + (!hasTokenSegments && !segment.isWhitespace && segment.text.trim()), + ) + +const buildStackedPronunciationSegments = (line, pronunciationLine) => { + const lineSegments = buildSegmentsFromLine(line) + const hasTokenSegments = lineSegments.some((segment) => segment.token) + const mainSegments = hasTokenSegments + ? lineSegments + : splitTextSegments(line?.value || '') + const pronunciationParts = buildPronunciationParts(pronunciationLine) + const pairableSegments = mainSegments.filter((segment) => + canPairPronunciationSegment(segment, hasTokenSegments), + ) + + if ( + !hasTokenSegments && + pronunciationParts.length > 0 && + pairableSegments.length > 0 && + pronunciationParts.length !== pairableSegments.length + ) { + return [ + { + text: line?.value || '', + token: null, + tokenIndex: -1, + isWhitespace: false, + pronunciation: pronunciationLine?.value?.trim() || '', + pronunciationSegment: null, + }, + ] + } + + const plainPronunciationParts = [] + const emphasisPronunciationParts = [] + for (const part of pronunciationParts) { + if (hasTokenSegments && isEmphasisRole(part.segment?.token)) { + emphasisPronunciationParts.push(part) + } else { + plainPronunciationParts.push(part) + } + } + + let pronunciationIndex = 0 + let plainPronunciationIndex = 0 + let emphasisPronunciationIndex = 0 + + const getPronunciationPart = (segment) => { + if (!hasTokenSegments) { + const part = pronunciationParts[pronunciationIndex] || null + pronunciationIndex += 1 + return part + } + + if (isEmphasisRole(segment.token)) { + const part = + emphasisPronunciationParts[emphasisPronunciationIndex] || null + emphasisPronunciationIndex += 1 + return part + } + + const part = plainPronunciationParts[plainPronunciationIndex] || null + plainPronunciationIndex += 1 + return part + } + + return mainSegments.map((segment) => { + if (!canPairPronunciationSegment(segment, hasTokenSegments)) { + return { ...segment, pronunciation: '' } + } + const pronunciationPart = getPronunciationPart(segment) + return { + ...segment, + pronunciation: pronunciationPart?.text || '', + pronunciationSegment: pronunciationPart?.segment || null, + } + }) +} + +export const KaraokeStackedLineRow = memo( + ({ + lineIndex, + line, + pronunciationLine, + pronunciationStyle, + nextLineStart, + className, + style, + tokenClassName, + waveCharacterClassName, + classes, + registerToken, + renderCharacterWave = true, + rowKey = 'main', + testId, + }) => { + const rowRef = useRef(null) + const getTokenRef = useStableTokenRef(registerToken) + const [isWrapped, setIsWrapped] = useState(false) + const segments = useMemo( + () => buildStackedPronunciationSegments(line, pronunciationLine), + [line, pronunciationLine], + ) + const mainWindows = useMemo( + () => resolveKaraokeTokenWindows(line, nextLineStart), + [line, nextLineStart], + ) + const pronunciationWindows = useMemo( + () => resolveKaraokeTokenWindows(pronunciationLine, null), + [pronunciationLine], + ) + const tokenRGB = useMemo( + () => (style?.color ? parseColorRGB(style.color) : [255, 255, 255]), + [style?.color], + ) + const pronunciationRGB = useMemo( + () => + pronunciationStyle?.color + ? parseColorRGB(pronunciationStyle.color) + : [255, 255, 255], + [pronunciationStyle?.color], + ) + const hasTimedTokens = mainWindows.some( + (window) => window?.start != null && window?.end != null, + ) + const lineStyle = useMemo(() => buildLineStyle(line, style), [line, style]) + + useLayoutEffect(() => { + const row = rowRef.current + if (!row) return undefined + + const updateWrappedState = () => { + const tokenRows = new Set( + Array.from(row.querySelectorAll('[data-stacked-token="true"]')).map( + (node) => node.offsetTop, + ), + ) + const wrapped = tokenRows.size > 1 + setIsWrapped((current) => (current === wrapped ? current : wrapped)) + } + + updateWrappedState() + const ResizeObserverConstructor = + typeof window !== 'undefined' ? window.ResizeObserver : null + if (!ResizeObserverConstructor) return undefined + const resizeObserver = new ResizeObserverConstructor(updateWrappedState) + resizeObserver.observe(row) + return () => resizeObserver.disconnect() + }, [segments]) + + return ( + + {segments.map((segment, idx) => { + const resolvedMainWindow = segment.token + ? mainWindows[segment.tokenIndex] + : null + const mainWindow = isTimedWindow(resolvedMainWindow) + ? resolvedMainWindow + : null + const mainKey = `${lineIndex}:${rowKey}:${segment.tokenIndex}:main` + const mainTokenData = segment.token + ? buildTokenData(segment.token, tokenRGB, line) + : null + const mainEmphasized = + isEmphasisRole(segment.token) || isEmphasisRole(line) + const mainText = segment.token ? ( + + {renderWaveText( + segment.text, + Boolean(mainWindow?.start != null && mainWindow?.end != null), + Boolean( + renderCharacterWave && + mainWindow?.start != null && + mainWindow?.end != null, + ), + mainEmphasized, + waveCharacterClassName, + )} + + ) : ( + + {segment.text} + + ) + + if (!segment.pronunciation) return mainText + + const pronunciationToken = segment.pronunciationSegment?.token + const resolvedPronunciationWindow = pronunciationToken + ? pronunciationWindows[segment.pronunciationSegment.tokenIndex] + : null + const pronunciationWindow = + mainWindow || + (isTimedWindow(resolvedPronunciationWindow) + ? resolvedPronunciationWindow + : null) + const pronunciationTokenIndex = + segment.pronunciationSegment?.tokenIndex >= 0 + ? segment.pronunciationSegment.tokenIndex + : segment.tokenIndex >= 0 + ? segment.tokenIndex + : idx + const pronunciationKey = `${lineIndex}:${rowKey}:${pronunciationTokenIndex}:pronunciation` + const pronunciationTokenData = pronunciationWindow + ? buildTokenData( + pronunciationToken || segment.token, + pronunciationRGB, + pronunciationLine || line, + ) + : null + const pronunciationEmphasized = Boolean( + isEmphasisRole(pronunciationToken) || + isEmphasisRole(segment.token) || + isEmphasisRole(pronunciationLine) || + isEmphasisRole(line), + ) + return ( + + {mainText} + + {renderWaveText( + segment.pronunciation, + Boolean(pronunciationWindow), + Boolean(renderCharacterWave && pronunciationWindow), + pronunciationEmphasized, + waveCharacterClassName, + )} + + + ) + })} + + ) + }, +) + +KaraokeStackedLineRow.displayName = 'KaraokeStackedLineRow' diff --git a/ui/src/audioplayer/LyricsPanel.jsx b/ui/src/audioplayer/LyricsPanel.jsx new file mode 100644 index 000000000..bc6cfe20d --- /dev/null +++ b/ui/src/audioplayer/LyricsPanel.jsx @@ -0,0 +1,902 @@ +import { alpha, makeStyles, useTheme } from '@material-ui/core/styles' +import clsx from 'clsx' +import React, { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react' +import { + buildKaraokeLines, + hasStructuredLyricContent, + hasUsableKaraokeTiming, +} from './lyrics' +import { KaraokeLineRow, KaraokeStackedLineRow } from './LyricsLineRows' +import { + KARAOKE_ANIMATION_MS, + KARAOKE_AUX_LINE_HEIGHT, + KARAOKE_DESKTOP_ACTIVE_LINE_ANCHOR_RATIO, + KARAOKE_EASING, + KARAOKE_INLINE_ACTIVE_LINE_ANCHOR_RATIO, + KARAOKE_LINE_ENTER_MS, + KARAOKE_LINE_LIFT_PX, + KARAOKE_LINE_MOTION_EASING, + KARAOKE_MANUAL_SCROLL_PAUSE_MS, + KARAOKE_SCROLLBAR_VISIBLE_MS, + KARAOKE_TRANSLATION_OPACITY, +} from './lyricsKaraokeConstants' +import { + animateScrollTop, + cancelScrollAnimation, + getAnchoredScrollTop, + getScrollEndPadding, +} from './lyricsScroll' +import useLyricsTimeline from './useLyricsTimeline' + +const KARAOKE_LAYER_OPACITY_TRANSITION = `opacity ${KARAOKE_ANIMATION_MS}ms ${KARAOKE_EASING}` +const KARAOKE_IDLE_LAYER_OPACITY = 0.49 +const KARAOKE_TRANSLATION_IDLE_OPACITY = + KARAOKE_IDLE_LAYER_OPACITY * KARAOKE_TRANSLATION_OPACITY + +const useStyles = makeStyles((theme) => ({ + root: { + height: '100%', + minHeight: 0, + display: 'flex', + flexDirection: 'column', + color: theme.palette.text.primary, + }, + inlineRoot: { + borderRadius: 'inherit', + background: 'transparent', + backdropFilter: 'blur(16px)', + WebkitBackdropFilter: 'blur(16px)', + }, + body: { + flex: 1, + minHeight: 0, + overflowY: 'auto', + overflowX: 'hidden', + padding: theme.spacing(4, 2.25, 3.25), + overscrollBehavior: 'contain', + scrollbarWidth: 'none', + msOverflowStyle: 'none', + maskImage: + 'linear-gradient(to bottom, transparent 0, rgba(0, 0, 0, 0.15) 12px, #000 40px, #000 calc(100% - 120px), rgba(0, 0, 0, 0.12) calc(100% - 48px), transparent 100%)', + WebkitMaskImage: + 'linear-gradient(to bottom, transparent 0, rgba(0, 0, 0, 0.15) 12px, #000 40px, #000 calc(100% - 120px), rgba(0, 0, 0, 0.12) calc(100% - 48px), transparent 100%)', + '&::-webkit-scrollbar': { + width: 0, + height: 0, + }, + }, + bodyTopFade: { + maskImage: + 'linear-gradient(to bottom, transparent 0, rgba(0, 0, 0, 0.15) 8px, #000 24px, #000 calc(100% - 120px), rgba(0, 0, 0, 0.12) calc(100% - 48px), transparent 100%)', + WebkitMaskImage: + 'linear-gradient(to bottom, transparent 0, rgba(0, 0, 0, 0.15) 8px, #000 24px, #000 calc(100% - 120px), rgba(0, 0, 0, 0.12) calc(100% - 48px), transparent 100%)', + }, + bodyUserScrolling: { + scrollbarWidth: 'thin', + msOverflowStyle: 'auto', + '&::-webkit-scrollbar': { + width: 8, + height: 8, + }, + '&::-webkit-scrollbar-thumb': { + backgroundColor: theme.palette.action.disabled, + borderRadius: 999, + }, + '&::-webkit-scrollbar-track': { + backgroundColor: 'transparent', + }, + }, + inlineBody: { + padding: theme.spacing(0.5, 1.25, 1.5), + textAlign: 'center', + }, + lines: { + display: 'flex', + flexDirection: 'column', + alignItems: 'stretch', + gap: theme.spacing(3), + }, + lineGroup: { + width: '100%', + borderRadius: theme.shape.borderRadius, + '--lyrics-main-current-color': + 'var(--lyrics-main-idle-color, currentColor)', + '--lyrics-pronunciation-current-color': + 'var(--lyrics-pronunciation-idle-color, currentColor)', + '--lyrics-translation-current-color': + 'var(--lyrics-translation-idle-color, currentColor)', + '--lyrics-layer-opacity': KARAOKE_IDLE_LAYER_OPACITY, + transform: 'translateY(0)', + transition: `background-color 150ms ${KARAOKE_LINE_MOTION_EASING}`, + '&[role="button"]:focus-visible': { + backgroundColor: alpha(theme.palette.text.primary, 0.055), + }, + '@media (hover: hover) and (pointer: fine)': { + '&[role="button"]:hover': { + backgroundColor: alpha(theme.palette.text.primary, 0.055), + }, + }, + '&[data-raised="true"][data-line-motion="line"]': { + transform: `translateY(-${KARAOKE_LINE_LIFT_PX}px)`, + transition: `transform ${KARAOKE_LINE_ENTER_MS}ms ${KARAOKE_LINE_MOTION_EASING}, background-color 150ms ${KARAOKE_LINE_MOTION_EASING}`, + }, + '&[data-raised="true"][data-line-motion="character"][data-character-wave="false"]': + { + transform: `translateY(-${KARAOKE_LINE_LIFT_PX}px)`, + }, + '&[data-active="true"]': { + '--lyrics-main-current-color': + 'var(--lyrics-main-active-color, var(--lyrics-main-idle-color, currentColor))', + '--lyrics-pronunciation-current-color': + 'var(--lyrics-pronunciation-active-color, var(--lyrics-pronunciation-idle-color, currentColor))', + '--lyrics-translation-current-color': + 'var(--lyrics-translation-active-color, var(--lyrics-translation-idle-color, currentColor))', + '--lyrics-layer-opacity': 1, + }, + '@media (prefers-reduced-motion: reduce)': { + transition: 'none', + transform: 'none', + }, + }, + waveCharacter: { + display: 'inline-block', + verticalAlign: 'baseline', + transform: 'translateY(0)', + transformOrigin: 'center bottom', + '@media (prefers-reduced-motion: reduce)': { + transform: 'none !important', + }, + }, + line: { + display: 'inline-block', + width: '100%', + maxWidth: '100%', + fontWeight: 700, + fontSize: 24, + lineHeight: 1.18, + overflowWrap: 'anywhere', + whiteSpace: 'pre-wrap', + letterSpacing: 0, + color: 'var(--lyrics-main-current-color, currentColor)', + WebkitTextFillColor: 'var(--lyrics-main-current-color, currentColor)', + '&[data-tokenized="false"]': { + opacity: 'var(--lyrics-layer-opacity)', + color: 'var(--lyrics-main-active-color, currentColor)', + WebkitTextFillColor: 'var(--lyrics-main-active-color, currentColor)', + transition: KARAOKE_LAYER_OPACITY_TRANSITION, + }, + '@media (prefers-reduced-motion: reduce)': { + transition: 'none', + }, + }, + inlineLine: { + fontSize: 24, + }, + auxLine: { + display: 'block', + marginTop: theme.spacing(0.8), + fontWeight: 600, + fontSize: 15, + lineHeight: KARAOKE_AUX_LINE_HEIGHT, + overflowWrap: 'anywhere', + whiteSpace: 'pre-wrap', + letterSpacing: 0, + opacity: KARAOKE_TRANSLATION_IDLE_OPACITY, + color: 'var(--lyrics-translation-active-color, currentColor)', + WebkitTextFillColor: 'var(--lyrics-translation-active-color, currentColor)', + transition: KARAOKE_LAYER_OPACITY_TRANSITION, + '@media (prefers-reduced-motion: reduce)': { + transition: 'none', + transform: 'none', + }, + }, + characterWaveAuxLine: { + transform: `translateY(-${KARAOKE_LINE_LIFT_PX}px)`, + transition: `${KARAOKE_LAYER_OPACITY_TRANSITION}, transform ${KARAOKE_LINE_ENTER_MS}ms ${KARAOKE_LINE_MOTION_EASING}`, + '@media (prefers-reduced-motion: reduce)': { + transition: 'none', + transform: 'none', + }, + }, + activeAuxLine: { + opacity: KARAOKE_TRANSLATION_OPACITY, + }, + stackedToken: { + display: 'inline-flex', + flexDirection: 'column', + alignItems: 'flex-start', + verticalAlign: 'top', + width: 'max-content', + minWidth: 0, + maxWidth: '100%', + overflowWrap: 'anywhere', + whiteSpace: 'pre-wrap', + paddingRight: theme.spacing(0.5), + marginBottom: theme.spacing(0.25), + }, + wrappedStackedLine: { + '& $stackedToken': { + marginBottom: theme.spacing(0.95), + }, + }, + stackedSpacer: { + display: 'inline', + whiteSpace: 'pre-wrap', + }, + stackedMainText: { + display: 'block', + lineHeight: 1.05, + maxWidth: '100%', + overflowWrap: 'anywhere', + whiteSpace: 'pre-wrap', + }, + stackedPronunciation: { + display: 'block', + marginTop: theme.spacing(0.15), + fontSize: 15, + lineHeight: 1.05, + fontWeight: 700, + maxWidth: '100%', + overflowWrap: 'anywhere', + whiteSpace: 'pre-wrap', + color: 'var(--lyrics-pronunciation-current-color, currentColor)', + WebkitTextFillColor: + 'var(--lyrics-pronunciation-current-color, currentColor)', + '&[data-timed="false"]': { + color: 'var(--lyrics-pronunciation-active-color, currentColor)', + WebkitTextFillColor: + 'var(--lyrics-pronunciation-active-color, currentColor)', + }, + '&[data-timed="true"]': { + transition: 'none', + }, + '@media (prefers-reduced-motion: reduce)': { + transition: 'none', + }, + }, + translationLine: { + fontWeight: 600, + }, + token: { + whiteSpace: 'pre-wrap', + overflowWrap: 'anywhere', + fontKerning: 'none', + fontVariantLigatures: 'none', + }, + voiceLanes: { + display: 'flex', + flexDirection: 'column', + alignItems: 'stretch', + gap: theme.spacing(0.7), + }, + secondaryVoiceLane: { + fontSize: 22, + }, + emptyState: { + flex: 1, + minHeight: 0, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + padding: theme.spacing(3), + color: alpha(theme.palette.text.primary, 0.68), + fontWeight: 600, + textAlign: 'center', + }, +})) + +const normalizeLineText = (value) => + String(value || '') + .normalize('NFKC') + .toLocaleLowerCase() + .replace(/[\p{P}\p{S}\s]+/gu, '') + +const shouldShowAuxLine = (mainLine, auxLine) => + Boolean( + auxLine?.value && + normalizeLineText(auxLine.value) !== normalizeLineText(mainLine?.value), + ) + +const finiteLineTime = (value) => { + if (value == null || value === '') return null + const number = Number(value) + return Number.isFinite(number) ? number : null +} + +const getLayerMatchWindow = (lines, index) => { + const line = lines[index] + if (!line) return { start: null, end: null } + return { + start: finiteLineTime(line.start), + end: finiteLineTime(line.end) ?? finiteLineTime(lines[index + 1]?.start), + } +} + +const buildUniqueLayerMap = (mainLines, layerLines) => { + if (!mainLines.length || !layerLines.length) return {} + + const candidates = [] + for (let layerIndex = 0; layerIndex < layerLines.length; layerIndex += 1) { + if (layerLines[layerIndex]?.renderable === false) continue + const layerWindow = getLayerMatchWindow(layerLines, layerIndex) + for (let mainIndex = 0; mainIndex < mainLines.length; mainIndex += 1) { + if (mainLines[mainIndex]?.renderable === false) continue + const mainWindow = getLayerMatchWindow(mainLines, mainIndex) + let score = null + + if (layerWindow.start != null && mainWindow.start != null) { + const mainEnd = mainWindow.end ?? mainWindow.start + const layerEnd = layerWindow.end ?? layerWindow.start + const overlap = + Math.min(mainEnd, layerEnd) - + Math.max(mainWindow.start, layerWindow.start) + const mainDuration = Math.max(0, mainEnd - mainWindow.start) + const maxDelta = Math.max(550, Math.min(1400, mainDuration + 420)) + const startDelta = Math.abs(layerWindow.start - mainWindow.start) + if (overlap < 0 && startDelta > maxDelta) continue + score = + startDelta + + Math.abs(layerIndex - mainIndex) * 30 + + (overlap < 0 ? 200 : 0) + } else if (layerIndex === mainIndex) { + score = 2000 + } + + if (score != null) { + candidates.push({ layerIndex, mainIndex, score }) + } + } + } + + candidates.sort( + (left, right) => + left.score - right.score || + left.layerIndex - right.layerIndex || + left.mainIndex - right.mainIndex, + ) + + const usedLayers = new Set() + const usedMainLines = new Set() + const result = {} + for (const candidate of candidates) { + if ( + usedLayers.has(candidate.layerIndex) || + usedMainLines.has(candidate.mainIndex) + ) { + continue + } + usedLayers.add(candidate.layerIndex) + usedMainLines.add(candidate.mainIndex) + result[candidate.mainIndex] = layerLines[candidate.layerIndex] + } + return result +} + +const getLineLanes = (line) => + Array.isArray(line?.lanes) && line.lanes.length > 0 ? line.lanes : [line] + +const buildSynchronizedTranslationLine = (_mainLine, translationLine) => { + if (!translationLine) return null + return { + ...translationLine, + tokens: [], + lanes: undefined, + } +} + +const buildLineGroupStyle = (canSeekLine, layerStyles) => ({ + cursor: canSeekLine ? 'pointer' : undefined, + '--lyrics-main-idle-color': layerStyles.main.color, + '--lyrics-main-active-color': + layerStyles.main['--lyrics-active-color'] || layerStyles.main.color, + '--lyrics-pronunciation-idle-color': layerStyles.pronunciation.color, + '--lyrics-pronunciation-active-color': + layerStyles.pronunciation['--lyrics-active-color'] || + layerStyles.pronunciation.color, + '--lyrics-translation-idle-color': layerStyles.translation.color, + '--lyrics-translation-active-color': + layerStyles.translation['--lyrics-active-color'] || + layerStyles.translation.color, +}) + +const usePrefersReducedMotion = () => { + const [prefersReducedMotion, setPrefersReducedMotion] = useState(false) + + useEffect(() => { + if (typeof window === 'undefined' || !window.matchMedia) return undefined + const mediaQuery = window.matchMedia('(prefers-reduced-motion: reduce)') + const updatePreference = () => + setPrefersReducedMotion(Boolean(mediaQuery.matches)) + updatePreference() + if (mediaQuery.addEventListener) { + mediaQuery.addEventListener('change', updatePreference) + return () => mediaQuery.removeEventListener('change', updatePreference) + } + mediaQuery.addListener(updatePreference) + return () => mediaQuery.removeListener(updatePreference) + }, []) + + return prefersReducedMotion +} + +const LyricsPanel = ({ + visible = true, + mainLyric, + translationLyric, + pronunciationLyric, + showTranslation, + showPronunciation, + audioInstance, + inline = false, + loading = false, + error = null, + labels = {}, +}) => { + const classes = useStyles() + const theme = useTheme() + const bodyRef = useRef(null) + const scrollAnimationRef = useRef(null) + const scrollbarTimerRef = useRef(null) + const manualScrollTimerRef = useRef(null) + const manualScrollUntilRef = useRef(0) + const [showScrollbar, setShowScrollbar] = useState(false) + const [autoScrollResumeKey, setAutoScrollResumeKey] = useState(0) + const [layoutVersion, setLayoutVersion] = useState(0) + const [hasTopFade, setHasTopFade] = useState(false) + const [scrollEndPadding, setScrollEndPadding] = useState(0) + const prefersReducedMotion = usePrefersReducedMotion() + const activeLineAnchorRatio = inline + ? KARAOKE_INLINE_ACTIVE_LINE_ANCHOR_RATIO + : KARAOKE_DESKTOP_ACTIVE_LINE_ANCHOR_RATIO + + const mainLines = useMemo(() => buildKaraokeLines(mainLyric), [mainLyric]) + const translationLines = useMemo( + () => buildKaraokeLines(translationLyric), + [translationLyric], + ) + const pronunciationLines = useMemo( + () => buildKaraokeLines(pronunciationLyric), + [pronunciationLyric], + ) + const hasTimedMainLines = useMemo( + () => hasUsableKaraokeTiming(mainLines), + [mainLines], + ) + const { + activeIndexes, + primaryIndex: activeIndex, + scrollTargetIndex, + registerLine, + registerToken, + getLineNode, + syncNow, + timeline, + } = useLyricsTimeline({ + lines: mainLines, + audioInstance, + visible: visible && hasTimedMainLines, + reducedMotion: prefersReducedMotion, + }) + const activeIndexSet = useMemo(() => new Set(activeIndexes), [activeIndexes]) + + const trByMainIndex = useMemo(() => { + if (!showTranslation || translationLines.length === 0) return {} + return buildUniqueLayerMap(mainLines, translationLines) + }, [mainLines, translationLines, showTranslation]) + + const prByMainIndex = useMemo(() => { + if (!showPronunciation || pronunciationLines.length === 0) return {} + return buildUniqueLayerMap(mainLines, pronunciationLines) + }, [mainLines, pronunciationLines, showPronunciation]) + + const colors = useMemo( + () => ({ + main: theme.palette.text.primary, + pronunciation: theme.palette.primary.main, + translation: + theme.palette.text.secondary || + theme.palette.secondary?.main || + theme.palette.text.primary, + }), + [theme], + ) + + const layerStyles = useMemo(() => { + const styleFor = (layer) => { + const sourceColor = colors[layer] + if (layer === 'translation') { + return { + opacity: 1, + color: sourceColor, + '--lyrics-active-color': sourceColor, + } + } + if (!hasTimedMainLines) { + return { + opacity: 1, + color: alpha(sourceColor, layer === 'main' ? 0.98 : 0.86), + } + } + const activeAlpha = layer === 'main' ? 0.98 : 0.78 + const pronunciationFadeRatio = 0.38 / 0.78 + const idleAlpha = activeAlpha * pronunciationFadeRatio + return { + opacity: 1, + color: alpha(sourceColor, idleAlpha), + '--lyrics-active-color': alpha(sourceColor, activeAlpha), + } + } + return { + main: styleFor('main'), + pronunciation: styleFor('pronunciation'), + translation: styleFor('translation'), + } + }, [colors, hasTimedMainLines]) + + const showScrollbarForManualScroll = useCallback(() => { + if (scrollbarTimerRef.current) { + window.clearTimeout(scrollbarTimerRef.current) + } + setShowScrollbar(true) + scrollbarTimerRef.current = window.setTimeout(() => { + setShowScrollbar(false) + scrollbarTimerRef.current = null + }, KARAOKE_SCROLLBAR_VISIBLE_MS) + }, []) + + const clearManualScrollTimer = useCallback(() => { + if (manualScrollTimerRef.current) { + window.clearTimeout(manualScrollTimerRef.current) + manualScrollTimerRef.current = null + } + }, []) + + const resumeAutoScroll = useCallback(() => { + clearManualScrollTimer() + manualScrollUntilRef.current = 0 + setAutoScrollResumeKey((current) => current + 1) + }, [clearManualScrollTimer]) + + const markManualScrollIntent = useCallback(() => { + cancelScrollAnimation(scrollAnimationRef) + clearManualScrollTimer() + manualScrollUntilRef.current = + performance.now() + KARAOKE_MANUAL_SCROLL_PAUSE_MS + manualScrollTimerRef.current = window.setTimeout(() => { + manualScrollTimerRef.current = null + resumeAutoScroll() + }, KARAOKE_MANUAL_SCROLL_PAUSE_MS) + showScrollbarForManualScroll() + }, [clearManualScrollTimer, resumeAutoScroll, showScrollbarForManualScroll]) + + const updateTopFade = useCallback(() => { + const body = bodyRef.current + setHasTopFade((current) => { + const next = Boolean(body && body.scrollTop > 1) + return current === next ? current : next + }) + }, []) + + useEffect( + () => () => { + if (scrollbarTimerRef.current) { + window.clearTimeout(scrollbarTimerRef.current) + } + if (manualScrollTimerRef.current) { + window.clearTimeout(manualScrollTimerRef.current) + } + cancelScrollAnimation(scrollAnimationRef) + }, + [], + ) + + useLayoutEffect(() => { + const body = bodyRef.current + if (!visible || !body) return undefined + const ResizeObserverConstructor = + typeof window !== 'undefined' ? window.ResizeObserver : null + if (!ResizeObserverConstructor) return undefined + const resizeObserver = new ResizeObserverConstructor(() => { + setLayoutVersion((current) => current + 1) + }) + resizeObserver.observe(body) + return () => resizeObserver.disconnect() + }, [visible]) + + useLayoutEffect(() => { + const body = bodyRef.current + if (!visible || !body) { + setScrollEndPadding(0) + return + } + const nextPadding = getScrollEndPadding(body, activeLineAnchorRatio) + setScrollEndPadding((current) => + current === nextPadding ? current : nextPadding, + ) + }, [activeLineAnchorRatio, layoutVersion, mainLines.length, visible]) + + useLayoutEffect(() => { + const body = bodyRef.current + if (!visible || !body) return + cancelScrollAnimation(scrollAnimationRef) + clearManualScrollTimer() + manualScrollUntilRef.current = 0 + body.scrollTop = 0 + setHasTopFade(false) + }, [clearManualScrollTimer, mainLyric, visible]) + + useEffect(() => { + if (!visible || !hasTimedMainLines || scrollTargetIndex < 0) { + cancelScrollAnimation(scrollAnimationRef) + return undefined + } + + const animFrameId = window.requestAnimationFrame(() => { + if (manualScrollUntilRef.current > performance.now()) return + const body = bodyRef.current + const targetNode = getLineNode(scrollTargetIndex) + if (!body || !targetNode) return + animateScrollTop({ + body, + targetTop: getAnchoredScrollTop( + body, + targetNode, + activeLineAnchorRatio, + ), + reducedMotion: prefersReducedMotion, + scrollAnimationRef, + }) + }) + + return () => window.cancelAnimationFrame(animFrameId) + }, [ + activeLineAnchorRatio, + autoScrollResumeKey, + getLineNode, + hasTimedMainLines, + layoutVersion, + prefersReducedMotion, + scrollTargetIndex, + visible, + ]) + + if (!visible) return null + + if (!hasStructuredLyricContent(mainLyric) || mainLines.length === 0) { + const message = loading + ? labels.loading || 'Loading lyrics' + : error + ? labels.unavailable || 'Lyrics unavailable' + : labels.empty || 'No lyrics available' + return ( +
event.stopPropagation() : undefined} + > +
+ {message} +
+
+ ) + } + + const seekToLine = (line) => { + if (!audioInstance || line.start == null) return + audioInstance.currentTime = line.start / 1000 + syncNow(line.start, true) + resumeAutoScroll() + } + + return ( +
event.stopPropagation() : undefined} + > +
+
0 + ? { paddingBottom: scrollEndPadding } + : undefined + } + > + {mainLines.map((line, idx) => { + if (line.renderable === false) return null + const trLine = trByMainIndex[idx] + const prLine = prByMainIndex[idx] + const mainNextLineStart = + timeline.windows[idx]?.nextTimedStart ?? null + const showTr = shouldShowAuxLine(line, trLine) + const showPr = shouldShowAuxLine(line, prLine) + const lineLanes = getLineLanes(line) + const usesCharacterRise = lineLanes.some( + (lane) => Array.isArray(lane?.tokens) && lane.tokens.length > 0, + ) + const canSeekLine = Boolean(audioInstance && line.start != null) + const isActiveLine = activeIndexSet.has(idx) + const renderCharacterWave = Boolean( + !prefersReducedMotion && hasTimedMainLines && isActiveLine, + ) + const isStaticLine = !hasTimedMainLines + return ( +
registerLine(idx, node) + : undefined + } + className={classes.lineGroup} + data-line-motion={usesCharacterRise ? 'character' : 'line'} + data-character-wave={renderCharacterWave ? 'true' : 'false'} + data-active={isStaticLine || isActiveLine ? 'true' : 'false'} + {...(isStaticLine + ? { + 'data-lifecycle': 'active', + 'data-highlight-active': 'true', + 'data-raised': 'false', + } + : {})} + aria-current={idx === activeIndex ? 'true' : undefined} + data-scroll-target={ + idx === scrollTargetIndex ? 'true' : 'false' + } + data-testid="lyrics-line-group" + style={buildLineGroupStyle(canSeekLine, layerStyles)} + role={canSeekLine ? 'button' : undefined} + tabIndex={canSeekLine ? 0 : undefined} + onClick={() => seekToLine(line)} + onMouseDown={ + canSeekLine + ? (event) => { + event.preventDefault() + } + : undefined + } + onKeyDown={(event) => { + if ( + canSeekLine && + (event.key === 'Enter' || event.key === ' ') + ) { + event.preventDefault() + seekToLine(line) + } + }} + > + {lineLanes.length > 1 ? ( +
+ {lineLanes.map((lane, laneIdx) => { + const laneClassName = clsx(classes.line, { + [classes.inlineLine]: inline, + [classes.secondaryVoiceLane]: laneIdx > 0, + }) + const rowKey = lane.key || `lane-${laneIdx}` + + return showPr && laneIdx === 0 ? ( + + ) : ( + + ) + })} +
+ ) : showPr ? ( + + ) : ( + + )} + {showTr && ( + + )} +
+ ) + })} +
+
+
+ ) +} + +export default LyricsPanel diff --git a/ui/src/audioplayer/LyricsPanel.test.jsx b/ui/src/audioplayer/LyricsPanel.test.jsx new file mode 100644 index 000000000..40bd9ea9b --- /dev/null +++ b/ui/src/audioplayer/LyricsPanel.test.jsx @@ -0,0 +1,710 @@ +import React from 'react' +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react' +import { ThemeProvider, createTheme } from '@material-ui/core/styles' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import LyricsPanel from './LyricsPanel' +import { KaraokeLineRow } from './LyricsLineRows' +import { + KARAOKE_ANIMATION_MS, + KARAOKE_CHARACTER_LIFT_PX, + KARAOKE_LINE_LIFT_PX, + KARAOKE_MANUAL_SCROLL_PAUSE_MS, + KARAOKE_TRANSLATION_OPACITY, +} from './lyricsKaraokeConstants' + +const theme = createTheme({ + palette: { + primary: { main: '#2266aa' }, + text: { primary: '#111111', secondary: '#778899' }, + }, +}) + +const panel = (props, selectedTheme = theme) => ( + + + +) + +const renderPanel = (props, selectedTheme = theme) => { + const view = render(panel(props, selectedTheme)) + return { + ...view, + rerenderPanel: (nextProps) => + view.rerender(panel(nextProps, selectedTheme)), + } +} + +const mainLyric = { + synced: true, + line: [{ start: 0, end: 1000, value: 'Main line' }], +} + +const createTokenizedLyric = (value, first, second) => ({ + synced: true, + line: [{ start: 0, end: 1000, value }], + cueLine: [ + { + index: 0, + start: 0, + end: 1000, + value, + cue: [ + { start: 0, end: 500, value: first, byteStart: 0, byteEnd: 3 }, + { start: 500, end: 1000, value: second, byteStart: 5, byteEnd: 8 }, + ], + }, + ], +}) + +const tokenizedMainLyric = createTokenizedLyric('Main line', 'Main', 'line') +const tokenizedPronunciationLyric = createTokenizedLyric( + 'mein lain', + 'mein', + 'lain', +) + +const voiceCue = (value, agentId, start, end) => ({ + index: 0, + start, + end, + value, + agentId, + cue: [{ start, end, value }], +}) + +const multiAgentLyric = { + synced: true, + agents: [ + { id: 'lead', role: 'main' }, + { id: 'all', role: 'group' }, + { id: 'echo', role: 'bg' }, + ], + line: [{ start: 1000, end: 4000, value: 'Lead all echo' }], + cueLine: [ + voiceCue('Lead', 'lead', 1000, 2000), + voiceCue('all', 'all', 1500, 2600), + voiceCue('echo', 'echo', 2200, 3400), + ], +} + +const createAudio = ({ currentTime = 0, duration = 10 } = {}) => { + const audio = new EventTarget() + audio.currentTime = currentTime + audio.duration = duration + audio.paused = true + audio.playbackRate = 1 + audio.seeking = false + return audio +} + +describe('', () => { + const originalMatchMedia = window.matchMedia + + beforeEach(() => { + vi.spyOn(window, 'requestAnimationFrame').mockImplementation(() => 0) + vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}) + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + window.matchMedia = originalMatchMedia + cleanup() + }) + + it('renders main, stacked pronunciation, and translation in layer order', () => { + renderPanel({ + mainLyric: tokenizedMainLyric, + pronunciationLyric: tokenizedPronunciationLyric, + translationLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: 'translation line' }], + }, + showPronunciation: true, + showTranslation: true, + }) + + const pronunciation = screen.getAllByTestId('lyrics-pronunciation-token') + expect(pronunciation).toHaveLength(2) + expect(pronunciation[0]).toHaveTextContent('mein') + expect(pronunciation[1]).toHaveTextContent('lain') + expect(screen.getByText('translation line')).toBeInTheDocument() + }) + + it('updates every pronunciation-only token with a distinct registry key', () => { + const audioInstance = createAudio() + renderPanel({ + mainLyric, + pronunciationLyric: tokenizedPronunciationLyric, + showPronunciation: true, + audioInstance, + }) + + const pronunciation = screen.getAllByTestId('lyrics-pronunciation-token') + act(() => { + audioInstance.currentTime = 0.75 + audioInstance.dispatchEvent(new Event('seeking')) + }) + + expect(pronunciation[0]).toHaveAttribute('data-lyrics-state', 'completed') + expect(pronunciation[1]).toHaveAttribute('data-lyrics-state', 'active') + }) + + it('renders incomplete pronunciation timing as static text', () => { + renderPanel({ + mainLyric, + pronunciationLyric: { + synced: true, + line: [{ value: 'mein lain' }], + cueLine: [ + { + index: 0, + value: 'mein lain', + cue: [{ value: 'mein' }, { value: 'lain' }], + }, + ], + }, + showPronunciation: true, + }) + + screen.getAllByTestId('lyrics-pronunciation-token').forEach((token) => { + expect(token).toHaveAttribute('data-timed', 'false') + expect(token).not.toHaveAttribute('aria-label') + }) + }) + + it('keeps token ref callbacks stable across equivalent rerenders', () => { + const registerToken = vi.fn() + const line = { + start: 0, + end: 1000, + value: 'Main', + tokens: [ + { + start: 0, + end: 1000, + value: 'Main', + byteStart: 0, + byteEnd: 3, + }, + ], + } + const row = () => ( + + ) + const { rerender } = render(row()) + + expect(registerToken).toHaveBeenCalledWith( + expect.any(String), + expect.any(Object), + expect.any(HTMLElement), + ) + registerToken.mockClear() + + rerender(row()) + + expect(registerToken).not.toHaveBeenCalled() + }) + + it('renders each translation line under only its closest main line', () => { + renderPanel({ + mainLyric: { + synced: true, + line: [ + { start: 0, end: 1000, value: 'First main line' }, + { start: 1000, end: 2000, value: 'Closest main line' }, + { start: 2000, end: 3000, value: 'Later main line' }, + ], + }, + translationLyric: { + synced: true, + line: [{ start: 1100, end: 2800, value: 'One translated line' }], + }, + showTranslation: true, + }) + + const translations = screen.getAllByText('One translated line') + expect(translations).toHaveLength(1) + expect( + translations[0].closest('[data-testid="lyrics-line-group"]'), + ).toHaveTextContent('Closest main line') + }) + + it('suppresses duplicate translations without shifting untimed indexes', () => { + renderPanel({ + mainLyric: { + synced: false, + line: [{ value: 'Hello' }, { value: 'World' }], + }, + translationLyric: { + synced: false, + line: [{ value: 'Hello' }, { value: 'Mundo' }], + }, + showTranslation: true, + }) + + const groups = screen.getAllByTestId('lyrics-line-group') + expect(groups[0]).toHaveTextContent('Hello') + expect(groups[0]).not.toHaveTextContent('Mundo') + expect(groups[1]).toHaveTextContent('World') + expect(groups[1]).toHaveTextContent('Mundo') + }) + + it('renders line-level pronunciation without inventing word timing', () => { + renderPanel({ + mainLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: '我总要给一些别的' }], + }, + pronunciationLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: 'wo zong yao gei yi xie bie de' }], + }, + showPronunciation: true, + audioInstance: { currentTime: 0.2, paused: true }, + }) + + expect(screen.getByText('我总要给一些别的')).toBeInTheDocument() + const pronunciation = screen.getByText('wo zong yao gei yi xie bie de') + const group = pronunciation.closest('[data-testid="lyrics-line-group"]') + expect(group).toHaveAttribute('data-active', 'true') + expect( + group.style.getPropertyValue('--lyrics-pronunciation-active-color'), + ).not.toBe('') + expect(pronunciation.style.color).toBe('') + }) + + it('uses one shared opacity animation for every static line layer', () => { + renderPanel({ + mainLyric, + pronunciationLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: 'main pronunciation' }], + }, + translationLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: 'translated line' }], + }, + showPronunciation: true, + showTranslation: true, + audioInstance: { currentTime: 0.5, paused: true }, + }) + + const mainRow = screen + .getByTestId('lyrics-line-group') + .querySelector('[data-tokenized]') + const translationRow = screen + .getByText('translated line') + .closest('[data-tokenized]') + const pronunciation = screen.getAllByTestId('lyrics-pronunciation-token') + + ;[mainRow, translationRow].forEach((row) => { + expect(row).toHaveAttribute('data-layer-animation', 'shared-opacity') + expect(row).toHaveAttribute('data-tokenized', 'false') + expect(row.style.opacity).toBe('') + expect(window.getComputedStyle(row).transition).toContain( + `opacity ${KARAOKE_ANIMATION_MS}ms`, + ) + }) + pronunciation.forEach((token) => + expect(token).toHaveAttribute('data-timed', 'false'), + ) + expect(window.getComputedStyle(translationRow).opacity).toBe( + String(KARAOKE_TRANSLATION_OPACITY), + ) + }) + + it('raises a line once and keeps it elevated after release', () => { + const { rerenderPanel } = renderPanel({ + mainLyric, + audioInstance: { currentTime: 0.5, paused: true }, + }) + + const group = screen.getByTestId('lyrics-line-group') + const activeStyle = window.getComputedStyle(group) + expect(group).toHaveAttribute('data-highlight-active', 'true') + expect(group).toHaveAttribute('data-raised', 'true') + expect(group).toHaveAttribute('data-line-motion', 'line') + expect(activeStyle.transform).toBe(`translateY(-${KARAOKE_LINE_LIFT_PX}px)`) + + rerenderPanel({ + mainLyric, + audioInstance: { currentTime: 1.1, paused: true }, + }) + + const releasedStyle = window.getComputedStyle(group) + expect(group).toHaveAttribute('data-highlight-active', 'false') + expect(group).toHaveAttribute('data-raised', 'true') + expect(releasedStyle.transform).toBe( + `translateY(-${KARAOKE_LINE_LIFT_PX}px)`, + ) + }) + + it('keeps every word-timed layer on the same rise and release lifecycle', () => { + const offsetPronunciationLyric = { + ...tokenizedPronunciationLyric, + cueLine: [ + { + ...tokenizedPronunciationLyric.cueLine[0], + cue: [ + { start: 100, end: 850, value: 'mein' }, + { start: 850, end: 1000, value: 'lain' }, + ], + }, + ], + } + const propsAt = (currentTime) => ({ + mainLyric: tokenizedMainLyric, + pronunciationLyric: offsetPronunciationLyric, + translationLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: 'translated line' }], + }, + showPronunciation: true, + showTranslation: true, + audioInstance: { currentTime, paused: true }, + }) + const { rerenderPanel } = renderPanel(propsAt(0.25)) + + const group = screen.getByTestId('lyrics-line-group') + const translation = screen + .getByText('translated line') + .closest('[data-tokenized]') + const mainToken = screen.getAllByTestId('lyrics-token')[0] + const pronunciationToken = screen.getAllByTestId( + 'lyrics-pronunciation-token', + )[0] + const mainCharacters = mainToken.querySelectorAll( + '[data-lyrics-character="true"]', + ) + const pronunciationCharacters = pronunciationToken.querySelectorAll( + '[data-lyrics-character="true"]', + ) + + expect(group).toHaveAttribute('data-line-motion', 'character') + expect(group).toHaveAttribute('data-character-wave', 'true') + expect(window.getComputedStyle(group).transform).toBe('translateY(0)') + expect(window.getComputedStyle(translation).transform).toBe( + `translateY(-${KARAOKE_LINE_LIFT_PX}px)`, + ) + expect(mainCharacters).toHaveLength(4) + expect(pronunciationCharacters).toHaveLength(4) + expect(mainCharacters[0].style.transform).toBe( + `translateY(-${KARAOKE_CHARACTER_LIFT_PX.toFixed(4)}px)`, + ) + expect(mainCharacters[3].style.transform).not.toBe( + mainCharacters[0].style.transform, + ) + expect(pronunciationCharacters[0].style.transform).toBe( + mainCharacters[0].style.transform, + ) + expect(mainToken.style.backgroundImage).toBe('none') + expect(mainCharacters[0].style.backgroundImage).toContain('linear-gradient') + + rerenderPanel(propsAt(1.1)) + + expect(group).toHaveAttribute('data-highlight-active', 'false') + expect(group).toHaveAttribute('data-raised', 'true') + expect(group).toHaveAttribute('data-character-wave', 'false') + expect( + group.querySelectorAll('[data-lyrics-character="true"]'), + ).toHaveLength(0) + expect(window.getComputedStyle(group).transform).toBe( + `translateY(-${KARAOKE_LINE_LIFT_PX}px)`, + ) + expect(window.getComputedStyle(translation).transform).toBe('') + const idleText = screen + .getAllByTestId('lyrics-token')[0] + .querySelector('[data-lyrics-wave-text="true"]') + expect(idleText.style.backgroundImage).toContain('linear-gradient') + }) + + it('keeps detailed grapheme markup on active lines without changing token layout', () => { + const values = ['First word', 'Second word', 'Active word', 'Fourth word'] + const lyricFor = (pronunciation = false) => ({ + synced: true, + line: values.map((value, index) => ({ + start: index * 1000, + end: (index + 1) * 1000, + value: pronunciation ? `spoken ${index}` : value, + })), + cueLine: values.map((value, index) => ({ + index, + start: index * 1000, + end: (index + 1) * 1000, + value: pronunciation ? `spoken ${index}` : value, + cue: (pronunciation ? `spoken ${index}` : value) + .split(' ') + .map((word, wordIndex, words) => ({ + start: index * 1000 + (wordIndex * 1000) / words.length, + end: index * 1000 + ((wordIndex + 1) * 1000) / words.length, + value: word, + })), + })), + }) + const propsAt = (currentTime) => ({ + mainLyric: lyricFor(), + pronunciationLyric: lyricFor(true), + showPronunciation: true, + audioInstance: { currentTime, paused: true }, + }) + const { rerenderPanel } = renderPanel(propsAt(2.5)) + + let groups = screen.getAllByTestId('lyrics-line-group') + expect( + groups.map((group) => + Boolean(group.querySelector('[data-lyrics-character="true"]')), + ), + ).toEqual([false, false, true, false]) + expect( + groups[2].querySelectorAll('[data-lyrics-wave-measure="true"]'), + ).not.toHaveLength(0) + const activeRow = groups[2].querySelector('[data-wrapped]') + const spacers = activeRow.querySelectorAll( + ':scope > span:not([data-stacked-token="true"])', + ) + expect(activeRow).toHaveAttribute('data-wrapped', 'false') + expect(spacers).not.toHaveLength(0) + spacers.forEach((spacer) => + expect(window.getComputedStyle(spacer).display).toBe('inline'), + ) + + rerenderPanel(propsAt(3.5)) + groups = screen.getAllByTestId('lyrics-line-group') + expect( + groups[2].querySelectorAll('[data-lyrics-character="true"]'), + ).toHaveLength(0) + expect( + groups[3].querySelectorAll('[data-lyrics-character="true"]'), + ).not.toHaveLength(0) + }) + + it('renders unsynced lyrics as static selectable text', () => { + renderPanel({ + mainLyric: { + synced: false, + line: [{ value: 'first plain line' }, { value: 'second plain line' }], + }, + }) + + const groups = screen.getAllByTestId('lyrics-line-group') + expect(groups).toHaveLength(2) + groups.forEach((group) => { + expect(group).toHaveAttribute('data-active', 'true') + expect(group).toHaveAttribute('data-lifecycle', 'active') + expect(group).toHaveAttribute('data-highlight-active', 'true') + expect(group).not.toHaveAttribute('aria-current') + expect(group).toHaveAttribute('data-scroll-target', 'false') + }) + }) + + it('tracks overlapping lines while selecting one primary line', () => { + renderPanel({ + mainLyric: { + synced: true, + line: [ + { start: 1000, end: 4000, value: 'Lead vocal' }, + { start: 2000, end: 3000, value: 'Answer vocal' }, + { start: 5000, end: 6000, value: 'Later vocal' }, + ], + }, + audioInstance: { currentTime: 2.5, paused: true }, + }) + + const groups = screen.getAllByTestId('lyrics-line-group') + expect(groups[0]).toHaveAttribute('data-active', 'true') + expect(groups[1]).toHaveAttribute('data-active', 'true') + expect(groups[2]).toHaveAttribute('data-active', 'false') + expect(groups[0]).not.toHaveAttribute('aria-current') + expect(groups[1]).toHaveAttribute('aria-current', 'true') + expect(groups[1]).toHaveAttribute('data-scroll-target', 'true') + }) + + it('keeps multi-agent cue lines in separate voice lanes', () => { + renderPanel({ + mainLyric: multiAgentLyric, + audioInstance: { currentTime: 2.5, paused: true }, + }) + + const lanes = screen.getAllByTestId('lyrics-voice-lane') + expect(lanes).toHaveLength(3) + expect(lanes[0]).toHaveTextContent('Lead') + expect(lanes[1]).toHaveTextContent('all') + expect(lanes[1].style.fontStyle).toBe('italic') + expect(lanes[2]).toHaveTextContent('echo') + expect(lanes[2].style.fontStyle).toBe('italic') + + const mainToken = lanes[0].querySelector('[data-testid="lyrics-token"]') + const emphasisToken = lanes[1].querySelector('[data-testid="lyrics-token"]') + const emphasisCharacter = emphasisToken.querySelector( + '[data-lyrics-character="true"]', + ) + expect(mainToken).toHaveTextContent('Lead') + expect(emphasisToken.style.paddingInlineEnd).not.toBe('') + expect(emphasisToken.style.marginInlineEnd).toBe( + `-${emphasisToken.style.paddingInlineEnd}`, + ) + expect(emphasisCharacter.style.paddingInlineEnd).toBe( + emphasisToken.style.paddingInlineEnd, + ) + }) + + it('seeks and synchronizes immediately when a line is activated', () => { + const audioInstance = { currentTime: 0 } + renderPanel({ + mainLyric: { + synced: true, + line: [{ start: 2300, end: 3200, value: 'Seek line' }], + }, + audioInstance, + }) + + const group = screen + .getByText('Seek line') + .closest('[data-testid="lyrics-line-group"]') + fireEvent.click(group) + expect(audioInstance.currentTime).toBe(2.3) + expect(group).toHaveAttribute('data-active', 'true') + + audioInstance.currentTime = 0 + fireEvent.keyDown(group, { key: 'Enter' }) + expect(audioInstance.currentTime).toBe(2.3) + }) + + it('pauses auto-scroll only for genuine manual scroll intent', async () => { + vi.useFakeTimers() + const requestAnimationFrameSpy = vi + .spyOn(window, 'requestAnimationFrame') + .mockImplementation(() => 0) + renderPanel({ + mainLyric, + audioInstance: { currentTime: 0.5, paused: true }, + }) + + const body = screen.getByTestId('lyrics-scroll-body') + const initialFrames = requestAnimationFrameSpy.mock.calls.length + fireEvent.wheel(body) + expect(body).toHaveAttribute('data-scrollbar-visible', 'true') + expect(requestAnimationFrameSpy).toHaveBeenCalledTimes(initialFrames) + + act(() => { + vi.advanceTimersByTime(KARAOKE_MANUAL_SCROLL_PAUSE_MS) + }) + await waitFor(() => { + expect(requestAnimationFrameSpy.mock.calls.length).toBeGreaterThan( + initialFrames, + ) + }) + }) + + it('resets scroll position when lyric content changes', () => { + const { rerenderPanel } = renderPanel({ mainLyric }) + const body = screen.getByTestId('lyrics-scroll-body') + body.scrollTop = 180 + + rerenderPanel({ + mainLyric: { + synced: true, + line: [{ start: 0, end: 1000, value: 'Different song' }], + }, + }) + expect(body.scrollTop).toBe(0) + }) + + it('respects reduced motion and empty states', async () => { + window.matchMedia = vi.fn().mockImplementation((query) => ({ + matches: query === '(prefers-reduced-motion: reduce)', + media: query, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + })) + const { unmount } = renderPanel({ mainLyric }) + await waitFor(() => { + expect(screen.getByTestId('lyrics-scroll-body')).toHaveAttribute( + 'data-reduced-motion', + 'true', + ) + }) + unmount() + + renderPanel({ mainLyric: null, loading: true }) + expect(screen.getByTestId('lyrics-empty-state')).toHaveTextContent( + 'Loading lyrics', + ) + }) + + it('dims translations relative to their semantic theme color', () => { + const sameTextTheme = createTheme({ + palette: { + primary: { main: '#ffffff' }, + text: { primary: '#ffffff', secondary: '#ffffff' }, + }, + }) + renderPanel( + { + mainLyric: { + synced: true, + line: [ + { start: 0, end: 1000, value: 'Active' }, + { start: 1000, end: 2000, value: 'Future' }, + ], + }, + translationLyric: { + synced: true, + line: [ + { start: 0, end: 1000, value: 'Activa' }, + { start: 1000, end: 2000, value: 'Futura' }, + ], + }, + showTranslation: true, + audioInstance: { currentTime: 0.5, paused: true }, + }, + sameTextTheme, + ) + + const groups = screen.getAllByTestId('lyrics-line-group') + const activeTranslation = screen + .getByText('Activa') + .closest('[data-tokenized]') + const futureTranslation = screen + .getByText('Futura') + .closest('[data-tokenized]') + groups.forEach((group) => { + expect( + group.style.getPropertyValue('--lyrics-translation-active-color'), + ).toBe('#ffffff') + }) + expect(groups[0]).toHaveAttribute('data-active', 'true') + expect(groups[1]).toHaveAttribute('data-active', 'false') + const layerOpacity = (group) => + window + .getComputedStyle(group) + .getPropertyValue('--lyrics-layer-opacity') + .trim() + expect(groups.map(layerOpacity)).toEqual(['1', '0.49']) + expect(window.getComputedStyle(activeTranslation).opacity).toBe( + String(KARAOKE_TRANSLATION_OPACITY), + ) + expect(Number(window.getComputedStyle(futureTranslation).opacity)).toBe( + 0.49 * KARAOKE_TRANSLATION_OPACITY, + ) + ;[activeTranslation, futureTranslation].forEach((translation) => + expect(window.getComputedStyle(translation).transition).toContain( + `opacity ${KARAOKE_ANIMATION_MS}ms`, + ), + ) + }) +}) diff --git a/ui/src/audioplayer/lyricsKaraokeConstants.js b/ui/src/audioplayer/lyricsKaraokeConstants.js new file mode 100644 index 000000000..db48b7bd9 --- /dev/null +++ b/ui/src/audioplayer/lyricsKaraokeConstants.js @@ -0,0 +1,29 @@ +export const KARAOKE_CLOCK_DRIFT_RESET_MS = 140 +export const KARAOKE_HIGHLIGHT_LEAD_MS = 120 +export const KARAOKE_ANIMATION_MS = 220 +export const KARAOKE_SCROLLBAR_VISIBLE_MS = 1400 +export const KARAOKE_MANUAL_SCROLL_PAUSE_MS = 2200 +export const KARAOKE_SCROLL_ANIMATION_MS = 300 +export const KARAOKE_LINE_ENTER_MS = KARAOKE_ANIMATION_MS +export const KARAOKE_LINE_LIFT_PX = 1.5 +export const KARAOKE_LINE_MOTION_EASING = 'cubic-bezier(0.25, 0.1, 0.25, 1)' +export const KARAOKE_CHARACTER_LIFT_PX = 1.5 +export const KARAOKE_CHARACTER_PHASE_SPREAD = 0.36 +export const KARAOKE_TRANSLATION_OPACITY = 0.62 +export const KARAOKE_LINE_RELEASE_MS = KARAOKE_ANIMATION_MS +export const KARAOKE_SCROLL_PRE_ROLL_MS = 320 +export const KARAOKE_DESKTOP_ACTIVE_LINE_ANCHOR_RATIO = 0.1 +export const KARAOKE_INLINE_ACTIVE_LINE_ANCHOR_RATIO = 0.42 +export const KARAOKE_SCROLL_SETTLE_PX = 2 +export const KARAOKE_AUX_LINE_HEIGHT = 1.18 +export const KARAOKE_EASING = 'cubic-bezier(0.22, 1, 0.36, 1)' +export const TOKEN_FUTURE_ALPHA = 0.34 +export const TOKEN_ACTIVE_ALPHA = 1 +export const TOKEN_WIPE_SOFT_SPREAD_PCT = 34 +export const TOKEN_WIPE_EDGE_PCT = 12 + +export const clamp = (value, min, max) => Math.min(max, Math.max(min, value)) +export const easeInOut = (value) => { + const clamped = clamp(value, 0, 1) + return clamped < 0.5 ? 2 * clamped * clamped : 1 - (-2 * clamped + 2) ** 2 / 2 +} diff --git a/ui/src/audioplayer/lyricsKaraokeStyles.js b/ui/src/audioplayer/lyricsKaraokeStyles.js new file mode 100644 index 000000000..6e751133f --- /dev/null +++ b/ui/src/audioplayer/lyricsKaraokeStyles.js @@ -0,0 +1,45 @@ +const EMPHASIS_ROLES = new Set([ + 'adlib', + 'backing', + 'backing vocals', + 'backing-vocals', + 'background', + 'background vocals', + 'background-vocals', + 'bg', + 'choir', + 'chorus', + 'group', + 'harmony', +]) + +const normalizeRole = (role) => + String(role || '') + .trim() + .toLowerCase() + +export const isEmphasisRole = (token) => + EMPHASIS_ROLES.has(normalizeRole(token?.role)) || + EMPHASIS_ROLES.has(normalizeRole(token?.agentRole)) + +export const buildEmphasisStyle = (token) => + isEmphasisRole(token) ? { fontStyle: 'italic' } : undefined + +export const parseColorRGB = (color) => { + const value = String(color || '').trim() + const hex = value.match(/^#([0-9a-f]{3}|[0-9a-f]{6})$/i) + if (hex) { + const raw = hex[1] + if (raw.length === 3) { + return raw.split('').map((part) => parseInt(part + part, 16)) + } + return [ + parseInt(raw.slice(0, 2), 16), + parseInt(raw.slice(2, 4), 16), + parseInt(raw.slice(4, 6), 16), + ] + } + + const rgb = value.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/) + return rgb ? [parseInt(rgb[1]), parseInt(rgb[2]), parseInt(rgb[3])] : null +} diff --git a/ui/src/audioplayer/lyricsScroll.js b/ui/src/audioplayer/lyricsScroll.js new file mode 100644 index 000000000..adf05d728 --- /dev/null +++ b/ui/src/audioplayer/lyricsScroll.js @@ -0,0 +1,73 @@ +import { + KARAOKE_SCROLL_ANIMATION_MS, + KARAOKE_SCROLL_SETTLE_PX, + clamp, + easeInOut, +} from './lyricsKaraokeConstants' + +export const cancelScrollAnimation = (scrollAnimationRef) => { + const animation = scrollAnimationRef.current + if (animation?.frameId != null) { + window.cancelAnimationFrame(animation.frameId) + } + scrollAnimationRef.current = null +} + +export const getScrollEndPadding = (body, anchorRatio) => { + const height = Number(body?.clientHeight) + if (!Number.isFinite(height) || height <= 0) return 0 + return Math.round(height * (1 - anchorRatio)) +} + +export const getAnchoredScrollTop = (body, targetNode, anchorRatio) => { + const bodyRect = body.getBoundingClientRect() + const targetRect = targetNode.getBoundingClientRect() + const activeAnchorTop = body.clientHeight * anchorRatio + const deltaWithinBody = targetRect.top - bodyRect.top - activeAnchorTop + const maxTop = Math.max(0, body.scrollHeight - body.clientHeight) + return clamp(body.scrollTop + deltaWithinBody, 0, maxTop) +} + +export const animateScrollTop = ({ + body, + targetTop, + reducedMotion, + scrollAnimationRef, +}) => { + cancelScrollAnimation(scrollAnimationRef) + + const startTop = body.scrollTop + const maxTop = Math.max(0, body.scrollHeight - body.clientHeight) + const nextTargetTop = clamp(targetTop, 0, maxTop) + const distance = nextTargetTop - startTop + if (Math.abs(distance) < KARAOKE_SCROLL_SETTLE_PX) return + + if (reducedMotion) { + body.scrollTop = nextTargetTop + return + } + + const startedAt = performance.now() + const animation = { frameId: null } + const step = () => { + if (scrollAnimationRef.current !== animation) return + + const progress = clamp( + (performance.now() - startedAt) / KARAOKE_SCROLL_ANIMATION_MS, + 0, + 1, + ) + body.scrollTop = + progress >= 1 ? nextTargetTop : startTop + distance * easeInOut(progress) + + if (progress < 1) { + animation.frameId = window.requestAnimationFrame(step) + return + } + + scrollAnimationRef.current = null + } + + animation.frameId = window.requestAnimationFrame(step) + scrollAnimationRef.current = animation +} diff --git a/ui/src/audioplayer/lyricsScroll.test.js b/ui/src/audioplayer/lyricsScroll.test.js new file mode 100644 index 000000000..f75a44fe7 --- /dev/null +++ b/ui/src/audioplayer/lyricsScroll.test.js @@ -0,0 +1,157 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { KARAOKE_SCROLL_ANIMATION_MS } from './lyricsKaraokeConstants' +import { + animateScrollTop, + cancelScrollAnimation, + getAnchoredScrollTop, + getScrollEndPadding, +} from './lyricsScroll' + +const createScrollableBody = (scrollTop = 0) => ({ + clientHeight: 200, + scrollHeight: 1000, + scrollTop, +}) + +const animate = (body, scrollAnimationRef, targetTop, reducedMotion = false) => + animateScrollTop({ + body, + targetTop, + reducedMotion, + scrollAnimationRef, + }) + +describe('lyrics scroll helpers', () => { + let animationFrames + let now + + beforeEach(() => { + animationFrames = [] + now = 0 + vi.spyOn(performance, 'now').mockImplementation(() => now) + vi.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => { + animationFrames.push(callback) + return animationFrames.length + }) + vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('calculates end padding and clamps anchored targets', () => { + expect(getScrollEndPadding({ clientHeight: 500 }, 0.42)).toBe(290) + expect(getScrollEndPadding({ clientHeight: 200 }, 0.5)).toBe(100) + expect(getScrollEndPadding({ clientHeight: 0 }, 0.5)).toBe(0) + expect(getScrollEndPadding(null, 0.5)).toBe(0) + const body = { + ...createScrollableBody(120), + getBoundingClientRect: () => ({ top: 50 }), + } + const target = { + getBoundingClientRect: () => ({ top: 310 }), + } + + // 120 + (310 - 50 - 200 * 0.4) = 300 + expect(getAnchoredScrollTop(body, target, 0.4)).toBe(300) + const clampedBody = { + ...createScrollableBody(20), + getBoundingClientRect: () => ({ top: 100 }), + } + const above = { + getBoundingClientRect: () => ({ top: -500 }), + } + const below = { + getBoundingClientRect: () => ({ top: 5000 }), + } + + expect(getAnchoredScrollTop(clampedBody, above, 0.4)).toBe(0) + expect(getAnchoredScrollTop(clampedBody, below, 0.4)).toBe(800) + }) + + it('stores a cancellable frame while animating scroll position', () => { + const body = createScrollableBody() + const scrollAnimationRef = { current: null } + window.requestAnimationFrame.mockImplementationOnce((callback) => { + animationFrames.push(callback) + return 0 + }) + + animate(body, scrollAnimationRef, 400) + + expect(scrollAnimationRef.current?.frameId).toBe(0) + + cancelScrollAnimation(scrollAnimationRef) + + expect(window.cancelAnimationFrame).toHaveBeenCalledWith(0) + expect(scrollAnimationRef.current).toBeNull() + }) + + it('updates scroll position immediately when reduced motion is enabled', () => { + const body = createScrollableBody() + const scrollAnimationRef = { current: null } + + animate(body, scrollAnimationRef, 400, true) + + expect(body.scrollTop).toBe(400) + expect(window.requestAnimationFrame).not.toHaveBeenCalled() + expect(scrollAnimationRef.current).toBeNull() + }) + + it('skips tiny scroll adjustments within the settle distance', () => { + const body = createScrollableBody() + const scrollAnimationRef = { current: null } + + animate(body, scrollAnimationRef, 1) + + expect(body.scrollTop).toBe(0) + expect(window.requestAnimationFrame).not.toHaveBeenCalled() + expect(scrollAnimationRef.current).toBeNull() + }) + + it('clears the current animation after natural completion', () => { + const body = createScrollableBody() + const scrollAnimationRef = { current: null } + + animate(body, scrollAnimationRef, 400) + + now = 1000 + animationFrames[0]() + + expect(body.scrollTop).toBe(400) + expect(scrollAnimationRef.current).toBeNull() + }) + + it('ignores stale frames after a newer scroll animation starts', () => { + const body = createScrollableBody() + const scrollAnimationRef = { current: null } + + animate(body, scrollAnimationRef, 400) + const firstAnimation = scrollAnimationRef.current + + animate(body, scrollAnimationRef, 500) + const secondAnimation = scrollAnimationRef.current + + expect(secondAnimation).not.toBe(firstAnimation) + + now = KARAOKE_SCROLL_ANIMATION_MS / 2 + animationFrames[0]() + + expect(body.scrollTop).toBe(0) + expect(scrollAnimationRef.current).toBe(secondAnimation) + expect(animationFrames).toHaveLength(2) + + animationFrames[1]() + + expect(body.scrollTop).toBe(250) + expect(scrollAnimationRef.current).toBe(secondAnimation) + expect(animationFrames).toHaveLength(3) + + now = KARAOKE_SCROLL_ANIMATION_MS + animationFrames[2]() + + expect(body.scrollTop).toBe(500) + expect(scrollAnimationRef.current).toBeNull() + }) +}) diff --git a/ui/src/audioplayer/lyricsSegments.js b/ui/src/audioplayer/lyricsSegments.js new file mode 100644 index 000000000..81b6eb5c7 --- /dev/null +++ b/ui/src/audioplayer/lyricsSegments.js @@ -0,0 +1,146 @@ +import { utf8ByteRangeToCodeUnitRange } from './lyrics' + +export const buildSegmentsFromLine = (line) => { + if (!line || !Array.isArray(line.tokens) || line.tokens.length === 0) { + return [{ text: line?.value || '', token: null, tokenIndex: -1 }] + } + + const text = line.value || '' + const exactSegments = (() => { + if (!text) return null + + const rangedTokens = line.tokens + .map((token, tokenIndex) => ({ + token, + tokenIndex, + range: utf8ByteRangeToCodeUnitRange( + text, + token?.byteStart, + token?.byteEnd, + ), + })) + .filter((entry) => entry.range != null) + + if ( + rangedTokens.length !== line.tokens.length || + rangedTokens.length === 0 + ) { + return null + } + + rangedTokens.sort( + (a, b) => + a.range.start - b.range.start || + a.range.end - b.range.end || + a.tokenIndex - b.tokenIndex, + ) + + const segments = [] + let cursor = 0 + for (const entry of rangedTokens) { + if (entry.range.start < cursor) return null + if (entry.range.start > cursor) { + segments.push({ + text: text.slice(cursor, entry.range.start), + token: null, + tokenIndex: -1, + }) + } + segments.push({ + text: entry.range.text, + token: entry.token, + tokenIndex: entry.tokenIndex, + }) + cursor = entry.range.end + } + + if (cursor < text.length) { + segments.push({ text: text.slice(cursor), token: null, tokenIndex: -1 }) + } + + return segments + })() + + if (exactSegments) return exactSegments + + const matchedSegments = [] + const fallbackSegments = [] + let cursor = 0 + let allMatched = text.length > 0 + let anyMatched = false + + const pushFallbackSeparatorIfNeeded = (nextTokenText) => { + if (fallbackSegments.length === 0) return + const prevText = fallbackSegments[fallbackSegments.length - 1].text || '' + if (!prevText || !nextTokenText) return + if (/\s$/.test(prevText) || /^\s/.test(nextTokenText)) return + if (/[A-Za-z0-9]$/.test(prevText) && /^[A-Za-z0-9]/.test(nextTokenText)) { + fallbackSegments.push({ text: ' ', token: null, tokenIndex: -1 }) + } + } + + for (let tokenIndex = 0; tokenIndex < line.tokens.length; tokenIndex += 1) { + const token = line.tokens[tokenIndex] + const tokenText = token.value || '' + if (!tokenText) continue + + pushFallbackSeparatorIfNeeded(tokenText) + fallbackSegments.push({ text: tokenText, token, tokenIndex }) + + if (!text) { + allMatched = false + continue + } + + const foundAt = text.indexOf(tokenText, cursor) + const lowerText = foundAt < 0 ? text.toLowerCase() : text + const lowerTokenText = foundAt < 0 ? tokenText.toLowerCase() : tokenText + const canReuseLowercaseOffsets = + lowerText.length === text.length && + lowerTokenText.length === tokenText.length + const normalizedFoundAt = + foundAt >= 0 + ? foundAt + : canReuseLowercaseOffsets + ? lowerText.indexOf(lowerTokenText, cursor) + : -1 + + if (normalizedFoundAt >= 0) { + anyMatched = true + if (normalizedFoundAt > cursor) { + matchedSegments.push({ + text: text.slice(cursor, normalizedFoundAt), + token: null, + tokenIndex: -1, + }) + } + const matchedTokenText = text.slice( + normalizedFoundAt, + normalizedFoundAt + tokenText.length, + ) + matchedSegments.push({ + text: matchedTokenText || tokenText, + token, + tokenIndex, + }) + cursor = normalizedFoundAt + tokenText.length + } else { + allMatched = false + } + } + + if (allMatched && anyMatched) { + if (cursor < text.length) { + matchedSegments.push({ + text: text.slice(cursor), + token: null, + tokenIndex: -1, + }) + } + return matchedSegments + } + + if (fallbackSegments.length > 0) return fallbackSegments + + return [{ text, token: null, tokenIndex: -1 }] +} diff --git a/ui/src/audioplayer/lyricsSegments.test.js b/ui/src/audioplayer/lyricsSegments.test.js new file mode 100644 index 000000000..209b42e5c --- /dev/null +++ b/ui/src/audioplayer/lyricsSegments.test.js @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest' +import { buildSegmentsFromLine } from './lyricsSegments' + +describe('buildSegmentsFromLine', () => { + it('does not reuse lowercase offsets when Unicode case folding expands text', () => { + const segments = buildSegmentsFromLine({ + value: '\u0130A', + tokens: [{ value: 'a' }], + }) + + expect(segments).toEqual([ + { text: 'a', token: { value: 'a' }, tokenIndex: 0 }, + ]) + }) +}) diff --git a/ui/src/audioplayer/lyricsTimeline.js b/ui/src/audioplayer/lyricsTimeline.js new file mode 100644 index 000000000..c7a7157a2 --- /dev/null +++ b/ui/src/audioplayer/lyricsTimeline.js @@ -0,0 +1,312 @@ +import { KARAOKE_SCROLL_PRE_ROLL_MS } from './lyricsKaraokeConstants' + +const finiteTime = (value) => { + if (value == null || value === '') return null + const number = Number(value) + return Number.isFinite(number) ? number : null +} + +const upperBound = (items, time, getTime) => { + let low = 0 + let high = items.length + while (low < high) { + const middle = (low + high) >>> 1 + if (getTime(items[middle]) <= time) low = middle + 1 + else high = middle + } + return low +} + +const sameIndexes = (left, right) => + left.length === right.length && + left.every((value, index) => value === right[index]) + +const getPrimaryActiveIndex = (timeline, indexes) => + indexes.reduce((primaryIndex, lineIndex) => { + if (primaryIndex < 0) return lineIndex + const primary = timeline?.windows?.[primaryIndex] + const candidate = timeline?.windows?.[lineIndex] + if ((candidate?.start ?? -Infinity) !== (primary?.start ?? -Infinity)) { + return (candidate?.start ?? -Infinity) > (primary?.start ?? -Infinity) + ? lineIndex + : primaryIndex + } + return lineIndex > primaryIndex ? lineIndex : primaryIndex + }, -1) + +const earliestTokenStart = (line) => { + const tokens = Array.isArray(line?.tokens) ? line.tokens : [] + let earliest = null + for (const token of tokens) { + const start = finiteTime(token?.start) + if (start != null && (earliest == null || start < earliest)) + earliest = start + } + return earliest +} + +const latestExplicitTokenEnd = (line) => { + const tokens = Array.isArray(line?.tokens) ? line.tokens : [] + let latest = null + for (const token of tokens) { + const end = finiteTime(token?.end) + if (end != null && (latest == null || end > latest)) latest = end + } + return latest +} + +export const buildLyricsTimeline = ( + lines, + { durationMs = null, fallbackLineDurationMs = 8000 } = {}, +) => { + const sourceLines = Array.isArray(lines) ? lines : [] + const starts = sourceLines.map( + (line) => finiteTime(line?.start) ?? earliestTokenStart(line), + ) + const nextTimedStarts = new Array(sourceLines.length).fill(null) + const timedEntries = starts + .map((start, lineIndex) => ({ start, lineIndex })) + .filter((entry) => entry.start != null) + .sort( + (left, right) => + left.start - right.start || left.lineIndex - right.lineIndex, + ) + let nextDistinctStart = null + for (let index = timedEntries.length - 1; index >= 0; ) { + const groupStart = timedEntries[index].start + let groupIndex = index + while (groupIndex >= 0 && timedEntries[groupIndex].start === groupStart) { + nextTimedStarts[timedEntries[groupIndex].lineIndex] = nextDistinctStart + groupIndex -= 1 + } + nextDistinctStart = groupStart + index = groupIndex + } + + const trackEnd = finiteTime(durationMs) + const windows = sourceLines.map((line, lineIndex) => { + const start = starts[lineIndex] + let end = finiteTime(line?.end) ?? latestExplicitTokenEnd(line) + if (end == null) end = nextTimedStarts[lineIndex] + if (end == null && start != null) { + end = start + fallbackLineDurationMs + if (trackEnd != null && trackEnd > start) { + end = Math.min(end, trackEnd) + } + } + const renderable = line?.renderable !== false + const intervalValid = start != null && end != null && end > start + const valid = renderable && intervalValid + return { + lineIndex, + start, + end, + renderable, + intervalValid, + valid, + preRollStart: + start == null ? null : Math.max(0, start - KARAOKE_SCROLL_PRE_ROLL_MS), + nextTimedStart: nextTimedStarts[lineIndex], + } + }) + + const events = windows + .filter((window) => window.valid) + .flatMap((window) => [ + { time: window.start, type: 'start', lineIndex: window.lineIndex }, + { time: window.end, type: 'end', lineIndex: window.lineIndex }, + ]) + .sort((left, right) => { + if (left.time !== right.time) return left.time - right.time + if (left.type !== right.type) return left.type === 'end' ? -1 : 1 + return left.lineIndex - right.lineIndex + }) + + const checkpoints = [] + const active = new Set() + events.forEach((event, index) => { + if (event.type === 'start') active.add(event.lineIndex) + else active.delete(event.lineIndex) + if ((index + 1) % 64 === 0) { + checkpoints.push({ + eventIndex: index + 1, + time: event.time, + active: Array.from(active), + }) + } + }) + + const scrollOrder = windows + .filter((window) => window.valid && window.preRollStart != null) + .sort( + (left, right) => + left.preRollStart - right.preRollStart || + left.start - right.start || + left.lineIndex - right.lineIndex, + ) + + return { windows, events, checkpoints, scrollOrder } +} + +export const getTimelineScrollTarget = (timeline, time) => { + const current = finiteTime(time) ?? 0 + const order = timeline?.scrollOrder || [] + const index = upperBound(order, current, (item) => item.preRollStart) - 1 + return index >= 0 ? order[index].lineIndex : -1 +} + +export class LyricTimelineCursor { + constructor(timeline) { + this.timeline = timeline + this.eventIndex = 0 + this.active = new Set() + this.lastTime = -Infinity + this.lastIndexes = [] + this.result = { + indexes: this.lastIndexes, + changed: false, + primaryIndex: -1, + } + } + + reset(timeline = this.timeline) { + this.timeline = timeline + this.eventIndex = 0 + this.active.clear() + this.lastTime = -Infinity + this.lastIndexes = [] + this.result.indexes = this.lastIndexes + this.result.changed = false + this.result.primaryIndex = -1 + } + + applyEvent(event) { + if (event.type === 'start') this.active.add(event.lineIndex) + else this.active.delete(event.lineIndex) + } + + seek(time) { + const events = this.timeline?.events || [] + const checkpoints = this.timeline?.checkpoints || [] + const checkpointIndex = + upperBound(checkpoints, time, (item) => item.time) - 1 + const checkpoint = + checkpointIndex >= 0 ? checkpoints[checkpointIndex] : null + this.active = new Set(checkpoint?.active || []) + this.eventIndex = checkpoint?.eventIndex || 0 + while ( + this.eventIndex < events.length && + events[this.eventIndex].time <= time + ) { + this.applyEvent(events[this.eventIndex]) + this.eventIndex += 1 + } + } + + update(time, forceSeek = false) { + const current = finiteTime(time) ?? 0 + const events = this.timeline?.events || [] + const previousEventIndex = this.eventIndex + if (forceSeek || current < this.lastTime) { + this.seek(current) + } else { + while ( + this.eventIndex < events.length && + events[this.eventIndex].time <= current + ) { + this.applyEvent(events[this.eventIndex]) + this.eventIndex += 1 + } + } + this.lastTime = current + if (!forceSeek && previousEventIndex === this.eventIndex) { + this.result.changed = false + return this.result + } + const indexes = Array.from(this.active).sort((a, b) => a - b) + const changed = !sameIndexes(indexes, this.lastIndexes) + this.lastIndexes = indexes + this.result.indexes = indexes + this.result.changed = changed + this.result.primaryIndex = getPrimaryActiveIndex(this.timeline, indexes) + return this.result + } +} + +export const resolveKaraokeTokenWindows = (line, lineEndFallback = null) => { + const tokens = Array.isArray(line?.tokens) ? line.tokens : [] + if (tokens.length === 0) return [] + + const lineStart = finiteTime(line?.start) + const lineEnd = finiteTime(line?.end) ?? finiteTime(lineEndFallback) + const hasLineWindow = + lineStart != null && lineEnd != null && lineEnd > lineStart + const tokenCount = tokens.length + const windows = new Array(tokenCount) + + for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex += 1) { + const token = tokens[tokenIndex] + const previousToken = tokenIndex > 0 ? tokens[tokenIndex - 1] : null + const nextToken = + tokenIndex + 1 < tokenCount ? tokens[tokenIndex + 1] : null + const estimatedStart = hasLineWindow + ? lineStart + ((lineEnd - lineStart) * tokenIndex) / tokenCount + : null + const estimatedEnd = hasLineWindow + ? lineStart + ((lineEnd - lineStart) * (tokenIndex + 1)) / tokenCount + : null + const previousEnd = + finiteTime(previousToken?.end) ?? finiteTime(previousToken?.start) + + let start = finiteTime(token?.start) + if (start == null) start = previousEnd ?? estimatedStart ?? lineStart + + let end = finiteTime(token?.end) + if (end == null) { + const nextStart = finiteTime(nextToken?.start) + const nextEstimatedStart = + hasLineWindow && tokenIndex + 1 < tokenCount + ? lineStart + ((lineEnd - lineStart) * (tokenIndex + 1)) / tokenCount + : null + end = nextStart ?? nextEstimatedStart ?? estimatedEnd ?? lineEnd + } + + if ( + tokenCount === 1 && + hasLineWindow && + (start == null || end == null || end <= start + 1) + ) { + start = lineStart + end = lineEnd + } + if (start != null && end != null && end < start) end = start + + const previousWindow = tokenIndex > 0 ? windows[tokenIndex - 1] : null + const visualStart = + start != null && previousWindow?.end != null + ? Math.max(start, previousWindow.end) + : start + const visualEnd = + end != null && visualStart != null && end <= visualStart + ? visualStart + 1 + : end + windows[tokenIndex] = { + start: visualStart, + end: visualEnd, + sourceStart: start, + sourceEnd: end, + } + } + + return windows +} + +export const tokenProgressAt = (window, time) => { + const current = finiteTime(time) ?? 0 + if (!window || window.start == null || window.end == null) return 0 + if (window.end <= window.start) return current >= window.start ? 1 : 0 + return Math.max( + 0, + Math.min(1, (current - window.start) / (window.end - window.start)), + ) +} diff --git a/ui/src/audioplayer/lyricsTimeline.test.js b/ui/src/audioplayer/lyricsTimeline.test.js new file mode 100644 index 000000000..f357d60eb --- /dev/null +++ b/ui/src/audioplayer/lyricsTimeline.test.js @@ -0,0 +1,196 @@ +import { describe, expect, it } from 'vitest' +import { + buildLyricsTimeline, + getTimelineScrollTarget, + LyricTimelineCursor, + resolveKaraokeTokenWindows, + tokenProgressAt, +} from './lyricsTimeline' +import { KARAOKE_SCROLL_PRE_ROLL_MS } from './lyricsKaraokeConstants' + +describe('lyricsTimeline', () => { + it('supports overlapping active lines and backward seeks', () => { + const timeline = buildLyricsTimeline([ + { start: 1000, end: 3000, tokens: [] }, + { start: 2000, end: 4000, tokens: [] }, + ]) + const cursor = new LyricTimelineCursor(timeline) + + expect(cursor.update(1500, true).indexes).toEqual([0]) + expect(cursor.update(2500).indexes).toEqual([0, 1]) + expect(cursor.update(3500).indexes).toEqual([1]) + expect(cursor.update(1500).indexes).toEqual([0]) + }) + + it('excludes zero and negative duration intervals', () => { + const timeline = buildLyricsTimeline([ + { start: 1000, end: 1000, tokens: [] }, + { start: 2000, end: 1500, tokens: [] }, + { start: 3000, end: 4000, tokens: [] }, + ]) + + expect(timeline.events).toEqual([ + { time: 3000, type: 'start', lineIndex: 2 }, + { time: 4000, type: 'end', lineIndex: 2 }, + ]) + }) + + it('uses the next timed line across untimed display rows', () => { + const timeline = buildLyricsTimeline([ + { start: 1000, tokens: [] }, + { value: 'untimed annotation', tokens: [] }, + { start: 5000, end: 6000, tokens: [] }, + ]) + + expect(timeline.windows[0].end).toBe(5000) + expect(timeline.windows[0].nextTimedStart).toBe(5000) + expect(timeline.windows[1].valid).toBe(false) + }) + + it('uses chronological timing independently from display order', () => { + const timeline = buildLyricsTimeline([ + { start: 3000, end: 5000, value: 'Displayed first', tokens: [] }, + { start: 1000, end: 4000, value: 'Displayed second', tokens: [] }, + { start: 2000, value: 'Displayed third', tokens: [] }, + ]) + const cursor = new LyricTimelineCursor(timeline) + + expect(timeline.windows[2].end).toBe(3000) + expect(cursor.update(2500, true)).toMatchObject({ + indexes: [1, 2], + primaryIndex: 2, + }) + expect(cursor.update(3500)).toMatchObject({ + indexes: [0, 1], + primaryIndex: 0, + }) + }) + + it('uses timed blank markers as boundaries without activating or scrolling to them', () => { + const timeline = buildLyricsTimeline([ + { + start: 1000, + value: 'Before pause', + tokens: [], + renderable: true, + }, + { start: 2000, value: '', tokens: [], renderable: false }, + { + start: 4000, + end: 5000, + value: 'After pause', + tokens: [], + renderable: true, + }, + ]) + const cursor = new LyricTimelineCursor(timeline) + + expect(timeline.windows[0].end).toBe(2000) + expect(timeline.windows[1]).toMatchObject({ + renderable: false, + intervalValid: true, + valid: false, + }) + expect(cursor.update(2500, true).indexes).toEqual([]) + expect(timeline.scrollOrder.map((window) => window.lineIndex)).toEqual([ + 0, 2, + ]) + }) + + it('bounds a final open line when media duration is unavailable', () => { + const timeline = buildLyricsTimeline([{ start: 7000, tokens: [] }], { + fallbackLineDurationMs: 8000, + }) + + expect(timeline.windows[0]).toMatchObject({ + start: 7000, + end: 15000, + valid: true, + }) + }) + + it('keeps a start-only line active until the next timestamp', () => { + const timeline = buildLyricsTimeline([ + { start: 1000, value: 'A lyric line', tokens: [] }, + { start: 5000, end: 6000, tokens: [] }, + ]) + const cursor = new LyricTimelineCursor(timeline) + + expect(timeline.windows[0].end).toBe(5000) + expect(cursor.update(4999, true).indexes).toEqual([0]) + expect(cursor.update(5000).indexes).toEqual([1]) + }) + + it('caps a final open line with track duration', () => { + const timeline = buildLyricsTimeline([{ start: 7000, tokens: [] }], { + durationMs: 10000, + }) + + expect(timeline.windows[0]).toMatchObject({ + start: 7000, + end: 10000, + valid: true, + }) + }) + + it('ignores a stale track duration that precedes the final line', () => { + const timeline = buildLyricsTimeline([{ start: 7000, tokens: [] }], { + durationMs: 5000, + fallbackLineDurationMs: 8000, + }) + + expect(timeline.windows[0]).toMatchObject({ + start: 7000, + end: 15000, + valid: true, + }) + }) + + it('seeks through checkpoints in timelines longer than 32 lines', () => { + const timeline = buildLyricsTimeline( + Array.from({ length: 40 }, (_, index) => ({ + start: index * 1000, + end: (index + 1) * 1000, + tokens: [], + })), + ) + const cursor = new LyricTimelineCursor(timeline) + + expect(timeline.checkpoints.length).toBeGreaterThan(0) + expect(cursor.update(35500, true)).toMatchObject({ + indexes: [35], + primaryIndex: 35, + }) + }) + + it('moves the scroll target at pre-roll without changing active state', () => { + const timeline = buildLyricsTimeline([ + { start: 1000, end: 1500, tokens: [] }, + { start: 3000, end: 3500, tokens: [] }, + ]) + + expect(getTimelineScrollTarget(timeline, 0)).toBe(-1) + expect( + getTimelineScrollTarget(timeline, 3000 - KARAOKE_SCROLL_PRE_ROLL_MS), + ).toBe(1) + }) + + it('precomputes token windows once and keeps progress deterministic', () => { + const line = { + start: 1000, + end: 3000, + tokens: [ + { start: 1000, value: 'one' }, + { start: 2000, end: 3000, value: 'two' }, + ], + } + const windows = resolveKaraokeTokenWindows(line) + + expect(windows).toEqual([ + { start: 1000, end: 2000, sourceStart: 1000, sourceEnd: 2000 }, + { start: 2000, end: 3000, sourceStart: 2000, sourceEnd: 3000 }, + ]) + expect(tokenProgressAt(windows[0], 1500)).toBe(0.5) + expect(tokenProgressAt(windows[0], 2500)).toBe(1) + }) +}) diff --git a/ui/src/audioplayer/useLyricsTimeline.js b/ui/src/audioplayer/useLyricsTimeline.js new file mode 100644 index 000000000..b83aa4b0d --- /dev/null +++ b/ui/src/audioplayer/useLyricsTimeline.js @@ -0,0 +1,713 @@ +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react' +import { + KARAOKE_CHARACTER_LIFT_PX, + KARAOKE_CHARACTER_PHASE_SPREAD, + KARAOKE_CLOCK_DRIFT_RESET_MS, + KARAOKE_HIGHLIGHT_LEAD_MS, + KARAOKE_LINE_RELEASE_MS, +} from './lyricsKaraokeConstants' +import { + buildLyricsTimeline, + getTimelineScrollTarget, + LyricTimelineCursor, + tokenProgressAt, +} from './lyricsTimeline' + +const mediaTimeMs = (audio) => { + const seconds = Number(audio?.currentTime) + return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : 0 +} + +const mediaDurationMs = (audio) => { + const seconds = Number(audio?.duration) + return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : null +} + +const sameIndexes = (left, right) => + left.length === right.length && + left.every((value, index) => value === right[index]) + +const setProgress = (record, value) => { + const next = Math.max(0, Math.min(1, value)) + if (record.progress != null && Math.abs(record.progress - next) < 0.001) + return + record.progress = next + record.node.style.setProperty('--lyrics-progress', String(next)) +} + +const smootherStep = (value) => + value * value * value * (value * (value * 6 - 15) + 10) + +const setCharacterLift = (record, progress) => { + const characters = record.characters || [] + if (!characters.length) return + + const riseSpan = Math.max(0.001, 1 - KARAOKE_CHARACTER_PHASE_SPREAD) + + characters.forEach(({ node, phase }) => { + const local = Math.max(0, Math.min(1, (progress - phase) / riseSpan)) + const lift = Math.max( + 0, + Math.min( + KARAOKE_CHARACTER_LIFT_PX, + KARAOKE_CHARACTER_LIFT_PX * smootherStep(local), + ), + ) + if (lift < 0.00005) { + if (node.style.transform) node.style.removeProperty('transform') + return + } + const nextTransform = `translateY(-${lift.toFixed(4)}px)` + if (node.style.transform !== nextTransform) { + node.style.transform = nextTransform + } + }) +} + +const setCharacterGradient = (record) => { + const tokenRect = record.node.getBoundingClientRect() + const tokenWidth = tokenRect.width || record.node.offsetWidth || 1 + if (!record.presentation?.gradient) return + + const measurements = record.characters.map((character) => ({ + character, + offset: character.node.getBoundingClientRect().left - tokenRect.left, + })) + + measurements.forEach(({ character, offset }) => { + const { node } = character + if ( + character.gradientWidth === tokenWidth && + character.gradientOffset === offset + ) { + return + } + character.gradientWidth = tokenWidth + character.gradientOffset = offset + node.style.color = 'transparent' + node.style.webkitTextFillColor = 'transparent' + node.style.backgroundImage = record.presentation.gradient + node.style.backgroundSize = `${tokenWidth}px 100%` + node.style.backgroundPosition = `${-offset}px 0` + node.style.backgroundRepeat = 'no-repeat' + node.style.backgroundClip = 'text' + node.style.webkitBackgroundClip = 'text' + }) +} + +const setTokenOpacity = (record, value) => { + const next = Math.max(0, Math.min(1, value)) + if (record.opacity != null && Math.abs(record.opacity - next) < 0.001) return + record.opacity = next + record.node.style.opacity = String(next) +} + +const setTokenActiveAlpha = (record, value) => { + const next = Math.max(0, Math.min(1, value)) + if (record.activeAlpha != null && Math.abs(record.activeAlpha - next) < 0.001) + return + record.activeAlpha = next + record.node.style.setProperty('--lyrics-token-active-alpha', String(next)) +} + +const setGradientTokenColor = (record) => { + const paintNode = record.paintNode || record.node + const paintsOnChild = paintNode !== record.node + + record.node.style.color = 'transparent' + record.node.style.webkitTextFillColor = 'transparent' + record.node.style.backgroundImage = paintsOnChild + ? 'none' + : record.presentation.gradient + record.node.style.backgroundSize = paintsOnChild ? 'auto' : '100% 100%' + record.node.style.backgroundClip = paintsOnChild ? 'border-box' : 'text' + record.node.style.webkitBackgroundClip = paintsOnChild ? 'border-box' : 'text' + + if (!paintsOnChild) return + + paintNode.style.color = 'transparent' + paintNode.style.webkitTextFillColor = 'transparent' + paintNode.style.backgroundImage = record.characters.length + ? 'none' + : record.presentation.gradient + paintNode.style.backgroundSize = '100% 100%' + paintNode.style.backgroundPosition = '0 0' + paintNode.style.backgroundRepeat = 'no-repeat' + paintNode.style.backgroundClip = 'text' + paintNode.style.webkitBackgroundClip = 'text' +} + +const getInactiveTokenAlpha = (presentation = {}) => + Math.min(1, Math.max(0, presentation.futureAlpha ?? 0.34)) + +const getActiveTokenAlpha = (presentation = {}) => + Math.min(1, Math.max(0, presentation.activeAlpha ?? 1)) + +const isGradientTokenState = (state) => + state === 'future' || + state === 'active' || + state === 'completed' || + state === 'release' || + state === 'inactive-past' + +const applyTokenState = (record, state, progress = 0) => { + const previousState = record.state + record.state = state + record.node.dataset.lyricsState = state + const presentation = record.presentation || {} + + if (!isGradientTokenState(previousState)) { + setGradientTokenColor(record) + } + + const isFuture = state === 'future' + const isPast = state === 'inactive-past' + const nextProgress = isFuture ? 0 : state === 'active' ? progress : 1 + setProgress(record, nextProgress) + setCharacterLift(record, nextProgress) + setTokenOpacity(record, 1) + setTokenActiveAlpha( + record, + isPast + ? getInactiveTokenAlpha(presentation) + : getActiveTokenAlpha(presentation), + ) +} + +const setTokenReleasePresentation = (record, progress) => { + const presentation = record.presentation || {} + const nextProgress = Math.max(0, Math.min(1, progress)) + if (record.state !== 'release') { + applyTokenState(record, 'completed', 1) + record.state = 'release' + record.node.dataset.lyricsState = 'release' + } + const activeAlpha = getActiveTokenAlpha(presentation) + const targetAlpha = getInactiveTokenAlpha(presentation) + setTokenOpacity(record, 1) + setTokenActiveAlpha( + record, + activeAlpha + (targetAlpha - activeAlpha) * nextProgress, + ) +} + +const resetToken = (record, state = 'future') => { + applyTokenState(record, state, 0) +} + +const setTokenPresentation = (record, time) => { + const progress = tokenProgressAt(record.window, time) + const state = + progress <= 0 ? 'future' : progress >= 1 ? 'completed' : 'active' + if (record.state !== state || state === 'active') { + applyTokenState(record, state, progress) + } +} + +const canListenToMedia = (audio) => + Boolean( + audio && + typeof audio.addEventListener === 'function' && + typeof audio.removeEventListener === 'function', + ) + +const useLyricsTimeline = ({ + lines, + audioInstance, + visible, + reducedMotion, +}) => { + const [durationMs, setDurationMs] = useState(() => + mediaDurationMs(audioInstance), + ) + + useEffect(() => { + const updateDuration = () => setDurationMs(mediaDurationMs(audioInstance)) + updateDuration() + if (!canListenToMedia(audioInstance)) return undefined + audioInstance.addEventListener('loadedmetadata', updateDuration) + audioInstance.addEventListener('durationchange', updateDuration) + return () => { + audioInstance.removeEventListener('loadedmetadata', updateDuration) + audioInstance.removeEventListener('durationchange', updateDuration) + } + }, [audioInstance]) + + const timeline = useMemo( + () => buildLyricsTimeline(lines, { durationMs }), + [durationMs, lines], + ) + const cursorRef = useRef(new LyricTimelineCursor(timeline)) + const lineNodesRef = useRef(new Map()) + const tokenRecordsRef = useRef(new Map()) + const tokenRecordsByNodeRef = useRef(new WeakMap()) + const tokenResizeObserverRef = useRef(null) + const lineTokenKeysRef = useRef(new Map()) + const releaseIndexesRef = useRef(new Set()) + const frameRef = useRef(0) + const resetInterpolationRef = useRef(null) + const lastPublishedIndexesRef = useRef([]) + const lastPublishedPrimaryIndexRef = useRef(-1) + const lastScrollTargetRef = useRef(-1) + const lastAppliedTimeRef = useRef(0) + const [activeState, setActiveState] = useState(() => ({ + indexes: [], + primaryIndex: -1, + })) + const [scrollTargetIndex, setScrollTargetIndex] = useState(-1) + + useLayoutEffect(() => { + const ResizeObserverConstructor = window.ResizeObserver + if (!ResizeObserverConstructor || reducedMotion) return undefined + + const observer = new ResizeObserverConstructor((entries) => { + entries.forEach(({ target }) => { + const record = tokenRecordsByNodeRef.current.get(target) + if (record) setCharacterGradient(record) + }) + }) + tokenResizeObserverRef.current = observer + tokenRecordsRef.current.forEach((record) => { + if (record.characters.length) observer.observe(record.node) + }) + return () => { + observer.disconnect() + tokenResizeObserverRef.current = null + } + }, [reducedMotion]) + + const setLineState = useCallback((lineIndex, phase) => { + const node = lineNodesRef.current.get(lineIndex) + if (!node) return + node.dataset.active = phase === 'active' ? 'true' : 'false' + node.dataset.lifecycle = phase + node.dataset.highlightActive = phase === 'active' ? 'true' : 'false' + node.dataset.raised = phase === 'idle' ? 'false' : 'true' + }, []) + + const resetLineTokens = useCallback((lineIndex, state = 'future') => { + lineTokenKeysRef.current.get(lineIndex)?.forEach((key) => { + const record = tokenRecordsRef.current.get(key) + if (record) resetToken(record, state) + }) + }, []) + + const updateLineTokens = useCallback((lineIndex, time) => { + lineTokenKeysRef.current.get(lineIndex)?.forEach((key) => { + const record = tokenRecordsRef.current.get(key) + if (record) setTokenPresentation(record, time) + }) + }, []) + + const updateLineReleaseTokens = useCallback((lineIndex, progress) => { + lineTokenKeysRef.current.get(lineIndex)?.forEach((key) => { + const record = tokenRecordsRef.current.get(key) + if (record) setTokenReleasePresentation(record, progress) + }) + }, []) + + const publishActiveState = useCallback((indexes, nextPrimaryIndex) => { + if ( + sameIndexes(lastPublishedIndexesRef.current, indexes) && + lastPublishedPrimaryIndexRef.current === nextPrimaryIndex + ) { + return + } + lastPublishedIndexesRef.current = indexes + lastPublishedPrimaryIndexRef.current = nextPrimaryIndex + setActiveState({ indexes, primaryIndex: nextPrimaryIndex }) + }, []) + + const publishScrollTarget = useCallback( + (time) => { + const nextIndex = getTimelineScrollTarget(timeline, time) + if (nextIndex === lastScrollTargetRef.current) return + lastScrollTargetRef.current = nextIndex + setScrollTargetIndex(nextIndex) + }, + [timeline], + ) + + const apply = useCallback( + (time, forceSeek = false) => { + const current = Number.isFinite(Number(time)) + ? Math.max(0, Number(time)) + : 0 + const lead = reducedMotion ? 0 : KARAOKE_HIGHLIGHT_LEAD_MS + lastAppliedTimeRef.current = current + const cursor = cursorRef.current + const previousIndexes = cursor.lastIndexes + const result = cursor.update(current, forceSeek) + + if (forceSeek) { + releaseIndexesRef.current.clear() + timeline.windows.forEach((window) => { + if (!window.valid || current < window.start) { + setLineState(window.lineIndex, 'idle') + resetLineTokens(window.lineIndex, 'future') + return + } + if (current >= window.end) { + if (current < window.end + KARAOKE_LINE_RELEASE_MS) { + releaseIndexesRef.current.add(window.lineIndex) + setLineState(window.lineIndex, 'release') + updateLineReleaseTokens( + window.lineIndex, + (current - window.end) / KARAOKE_LINE_RELEASE_MS, + ) + } else { + setLineState(window.lineIndex, 'past') + resetLineTokens(window.lineIndex, 'inactive-past') + } + } + }) + } else if (result.changed) { + previousIndexes.forEach((lineIndex) => { + if (cursor.active.has(lineIndex)) return + const window = timeline.windows[lineIndex] + if ( + window?.valid && + current >= window.end && + current < window.end + KARAOKE_LINE_RELEASE_MS + ) { + releaseIndexesRef.current.add(lineIndex) + setLineState(lineIndex, 'release') + updateLineReleaseTokens(lineIndex, 0) + } else { + setLineState(lineIndex, 'past') + resetLineTokens(lineIndex, 'inactive-past') + } + }) + } + + result.indexes.forEach((lineIndex) => { + releaseIndexesRef.current.delete(lineIndex) + setLineState(lineIndex, 'active') + updateLineTokens(lineIndex, current + lead) + }) + + releaseIndexesRef.current.forEach((lineIndex) => { + const window = timeline.windows[lineIndex] + if ( + !window?.valid || + current >= window.end + KARAOKE_LINE_RELEASE_MS || + current < window.end + ) { + releaseIndexesRef.current.delete(lineIndex) + const isPast = current >= (window?.end ?? Infinity) + setLineState(lineIndex, isPast ? 'past' : 'idle') + resetLineTokens(lineIndex, isPast ? 'inactive-past' : 'future') + return + } + updateLineReleaseTokens( + lineIndex, + (current - window.end) / KARAOKE_LINE_RELEASE_MS, + ) + }) + + if (result.changed || forceSeek) { + publishActiveState(result.indexes, result.primaryIndex) + } + publishScrollTarget(current) + return result + }, + [ + publishActiveState, + publishScrollTarget, + reducedMotion, + resetLineTokens, + setLineState, + timeline.windows, + updateLineReleaseTokens, + updateLineTokens, + ], + ) + + const registerLine = useCallback( + (lineIndex, node) => { + if (!node) { + lineNodesRef.current.delete(lineIndex) + return + } + lineNodesRef.current.set(lineIndex, node) + const window = timeline.windows[lineIndex] + const time = lastAppliedTimeRef.current + if (cursorRef.current.active.has(lineIndex)) { + setLineState(lineIndex, 'active') + } else if ( + window?.valid && + time >= window.end && + time < window.end + KARAOKE_LINE_RELEASE_MS + ) { + setLineState(lineIndex, 'release') + } else if (window?.valid && time >= window.end) { + setLineState(lineIndex, 'past') + } else { + setLineState(lineIndex, 'idle') + } + }, + [setLineState, timeline.windows], + ) + + const registerToken = useCallback( + (key, descriptor, node) => { + const existing = tokenRecordsRef.current.get(key) + if (!node) { + if (existing) { + tokenResizeObserverRef.current?.unobserve(existing.node) + tokenRecordsByNodeRef.current.delete(existing.node) + tokenRecordsRef.current.delete(key) + const keys = lineTokenKeysRef.current.get(existing.lineIndex) + keys?.delete(key) + if (keys?.size === 0) { + lineTokenKeysRef.current.delete(existing.lineIndex) + } + } + return + } + + const characterNodes = reducedMotion + ? [] + : Array.from(node.querySelectorAll('[data-lyrics-character="true"]')) + const characterCount = characterNodes.length + const characters = characterNodes.map((characterNode, index) => ({ + node: characterNode, + phase: + characterCount <= 1 + ? 0 + : (index / (characterCount - 1)) * KARAOKE_CHARACTER_PHASE_SPREAD, + })) + const record = { + key, + node, + paintNode: node.querySelector('[data-lyrics-wave-text="true"]'), + lineIndex: descriptor.lineIndex, + window: descriptor.window, + presentation: descriptor.presentation, + characters, + progress: null, + opacity: null, + activeAlpha: null, + state: null, + } + setCharacterGradient(record) + tokenRecordsByNodeRef.current.set(node, record) + if (characters.length) tokenResizeObserverRef.current?.observe(node) + tokenRecordsRef.current.set(key, record) + if (!lineTokenKeysRef.current.has(record.lineIndex)) { + lineTokenKeysRef.current.set(record.lineIndex, new Set()) + } + lineTokenKeysRef.current.get(record.lineIndex).add(key) + + if (cursorRef.current.active.has(record.lineIndex)) { + setTokenPresentation( + record, + lastAppliedTimeRef.current + + (reducedMotion ? 0 : KARAOKE_HIGHLIGHT_LEAD_MS), + ) + } else { + const lineWindow = timeline.windows[record.lineIndex] + if ( + lineWindow?.valid && + lastAppliedTimeRef.current >= lineWindow.end && + lastAppliedTimeRef.current < lineWindow.end + KARAOKE_LINE_RELEASE_MS + ) { + setTokenReleasePresentation( + record, + (lastAppliedTimeRef.current - lineWindow.end) / + KARAOKE_LINE_RELEASE_MS, + ) + } else { + resetToken( + record, + lineWindow?.valid && lastAppliedTimeRef.current >= lineWindow.end + ? 'inactive-past' + : 'future', + ) + } + } + }, + [reducedMotion, timeline.windows], + ) + + const getLineNode = useCallback( + (lineIndex) => lineNodesRef.current.get(lineIndex) || null, + [], + ) + + useLayoutEffect(() => { + lineNodesRef.current.forEach((node) => { + node.dataset.active = 'false' + node.dataset.lifecycle = 'idle' + node.dataset.highlightActive = 'false' + node.dataset.raised = 'false' + }) + tokenRecordsRef.current.forEach((record) => resetToken(record)) + cursorRef.current = new LyricTimelineCursor(timeline) + releaseIndexesRef.current.clear() + lastPublishedIndexesRef.current = [] + lastPublishedPrimaryIndexRef.current = -1 + lastScrollTargetRef.current = -1 + setActiveState({ indexes: [], primaryIndex: -1 }) + setScrollTargetIndex(-1) + apply(mediaTimeMs(audioInstance), true) + }, [apply, audioInstance, timeline]) + + useEffect(() => { + if ( + !audioInstance || + !timeline.events.length || + !canListenToMedia(audioInstance) + ) { + return undefined + } + const seek = () => { + resetInterpolationRef.current?.() + apply(mediaTimeMs(audioInstance), true) + } + audioInstance.addEventListener('seeking', seek) + audioInstance.addEventListener('seeked', seek) + return () => { + audioInstance.removeEventListener('seeking', seek) + audioInstance.removeEventListener('seeked', seek) + } + }, [apply, audioInstance, timeline.events.length]) + + useEffect(() => { + if (!audioInstance || !timeline.events.length || !visible) return undefined + if (!canListenToMedia(audioInstance)) { + apply(mediaTimeMs(audioInstance), true) + return undefined + } + + let cancelled = false + let anchorAudioMs = mediaTimeMs(audioInstance) + let anchorPerfMs = performance.now() + let lastFrameTime = anchorAudioMs + + const stop = () => { + if (frameRef.current) window.cancelAnimationFrame(frameRef.current) + frameRef.current = 0 + } + + const resetAnchor = () => { + anchorAudioMs = mediaTimeMs(audioInstance) + anchorPerfMs = performance.now() + lastFrameTime = anchorAudioMs + } + resetInterpolationRef.current = resetAnchor + + const readInterpolatedTime = () => { + const observed = mediaTimeMs(audioInstance) + const now = performance.now() + const rate = Number(audioInstance.playbackRate) + const canInterpolate = + !audioInstance.paused && + !audioInstance.seeking && + Number.isFinite(rate) && + rate > 0 + if (!canInterpolate) { + anchorAudioMs = observed + anchorPerfMs = now + lastFrameTime = observed + return observed + } + const predicted = anchorAudioMs + (now - anchorPerfMs) * rate + const drift = observed - predicted + let current = + Math.abs(drift) > KARAOKE_CLOCK_DRIFT_RESET_MS ? observed : predicted + if (Math.abs(drift) > KARAOKE_CLOCK_DRIFT_RESET_MS) { + anchorAudioMs = observed + anchorPerfMs = now + } + const backwards = lastFrameTime - current + if (backwards > 0) { + current = lastFrameTime + } + lastFrameTime = Math.max(0, current) + return lastFrameTime + } + + const tick = () => { + if ( + cancelled || + audioInstance.paused || + window.document.visibilityState !== 'visible' + ) { + stop() + return + } + apply(readInterpolatedTime()) + frameRef.current = window.requestAnimationFrame(tick) + } + + const start = () => { + if ( + frameRef.current || + audioInstance.paused || + window.document.visibilityState !== 'visible' + ) { + return + } + resetAnchor() + frameRef.current = window.requestAnimationFrame(tick) + } + + const syncAndStop = () => { + stop() + apply(mediaTimeMs(audioInstance), true) + } + + const visibility = () => { + if (window.document.visibilityState !== 'visible') { + stop() + return + } + apply(mediaTimeMs(audioInstance), true) + start() + } + + audioInstance.addEventListener('play', start) + audioInstance.addEventListener('pause', syncAndStop) + window.document.addEventListener('visibilitychange', visibility) + apply(mediaTimeMs(audioInstance), true) + start() + + return () => { + cancelled = true + stop() + if (resetInterpolationRef.current === resetAnchor) { + resetInterpolationRef.current = null + } + audioInstance.removeEventListener('play', start) + audioInstance.removeEventListener('pause', syncAndStop) + window.document.removeEventListener('visibilitychange', visibility) + } + }, [apply, audioInstance, timeline.events.length, visible]) + + const syncNow = useCallback( + (time = mediaTimeMs(audioInstance), forceSeek = true) => + apply(time, forceSeek), + [apply, audioInstance], + ) + + return { + activeIndexes: activeState.indexes, + primaryIndex: activeState.primaryIndex, + scrollTargetIndex, + registerLine, + registerToken, + getLineNode, + syncNow, + timeline, + } +} + +export default useLyricsTimeline diff --git a/ui/src/audioplayer/useLyricsTimeline.test.js b/ui/src/audioplayer/useLyricsTimeline.test.js new file mode 100644 index 000000000..84f45918c --- /dev/null +++ b/ui/src/audioplayer/useLyricsTimeline.test.js @@ -0,0 +1,456 @@ +import { act } from '@testing-library/react' +import { renderHook } from '@testing-library/react-hooks' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { KARAOKE_CHARACTER_LIFT_PX } from './lyricsKaraokeConstants' +import useLyricsTimeline from './useLyricsTimeline' + +const createAudio = ({ + currentTime = 0, + duration = 10, + paused = true, + playbackRate = 1, +} = {}) => { + const target = new EventTarget() + target.currentTime = currentTime + target.duration = duration + target.paused = paused + target.playbackRate = playbackRate + target.seeking = false + return target +} + +const lines = [ + { + start: 0, + end: 1000, + tokens: [ + { start: 0, end: 500, value: 'first' }, + { start: 500, end: 1000, value: 'second' }, + ], + }, + { start: 1000, end: 2000, tokens: [] }, +] + +const presentation = { + rgb: [255, 255, 255], + futureAlpha: 0.34, + activeAlpha: 1, + futureColor: 'rgba(255, 255, 255, 0.34)', + doneColor: 'rgba(255, 255, 255, 1)', + gradient: 'linear-gradient(90deg, white, transparent)', + useCrossfade: false, +} + +const createTokenNode = (text = '') => { + const tokenNode = document.createElement('span') + Array.from(text).forEach((character) => { + const node = document.createElement('span') + node.dataset.lyricsCharacter = 'true' + node.textContent = character + tokenNode.appendChild(node) + }) + return tokenNode +} + +const registerToken = (result, key, window, text = '') => { + const tokenNode = createTokenNode(text) + act(() => { + result.current.registerToken( + key, + { lineIndex: 0, window, presentation }, + tokenNode, + ) + }) + return tokenNode +} + +const syncNow = (result, time) => { + act(() => { + result.current.syncNow(time, true) + }) +} + +const renderTimeline = ({ + audio = createAudio(), + sourceLines = lines, + reducedMotion = false, +} = {}) => + renderHook(() => + useLyricsTimeline({ + lines: sourceLines, + audioInstance: audio, + visible: true, + reducedMotion, + }), + ) + +describe('useLyricsTimeline', () => { + beforeEach(() => { + vi.spyOn(window, 'requestAnimationFrame').mockImplementation(() => 1) + vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('synchronizes paused media without starting an animation loop', () => { + const audio = createAudio({ currentTime: 0.25, paused: true }) + const { result } = renderTimeline({ audio }) + const lineNode = document.createElement('div') + const tokenNode = createTokenNode() + + act(() => { + result.current.registerLine(0, lineNode) + result.current.registerToken( + '0:main:0', + { + lineIndex: 0, + window: { start: 0, end: 500 }, + presentation, + }, + tokenNode, + ) + }) + + expect(result.current.activeIndexes).toEqual([0]) + expect(lineNode.dataset.active).toBe('true') + expect(tokenNode.dataset.lyricsState).toBe('active') + expect( + Number(tokenNode.style.getPropertyValue('--lyrics-progress')), + ).toBeCloseTo(0.74, 2) + expect(window.requestAnimationFrame).not.toHaveBeenCalled() + }) + + it('recomputes token state immediately after seeking backward', () => { + const audio = createAudio({ currentTime: 0.75, paused: true }) + const { result } = renderTimeline({ audio, reducedMotion: true }) + const first = document.createElement('span') + const second = document.createElement('span') + + act(() => { + result.current.registerToken( + '0:first', + { + lineIndex: 0, + window: { start: 0, end: 500 }, + presentation, + }, + first, + ) + result.current.registerToken( + '0:second', + { + lineIndex: 0, + window: { start: 500, end: 1000 }, + presentation, + }, + second, + ) + }) + + expect(first.dataset.lyricsState).toBe('completed') + expect(second.dataset.lyricsState).toBe('active') + + act(() => { + audio.currentTime = 0.1 + audio.dispatchEvent(new Event('seeking')) + }) + + expect(first.dataset.lyricsState).toBe('active') + expect(second.dataset.lyricsState).toBe('future') + }) + + it('tracks overlapping line intervals as an active set', () => { + const audio = createAudio({ currentTime: 2.5, paused: true }) + const overlappingLines = [ + { start: 1000, end: 4000, tokens: [] }, + { start: 2000, end: 3000, tokens: [] }, + { start: 5000, end: 6000, tokens: [] }, + ] + const { result } = renderTimeline({ audio, sourceLines: overlappingLines }) + + expect(result.current.activeIndexes).toEqual([0, 1]) + expect(result.current.primaryIndex).toBe(1) + }) + + it('publishes the most recently started line when display order differs', () => { + const audio = createAudio({ currentTime: 2.5, paused: true }) + const displayOrderLines = [ + { start: 2000, end: 4000, tokens: [] }, + { start: 1000, end: 4000, tokens: [] }, + ] + const { result } = renderTimeline({ audio, sourceLines: displayOrderLines }) + + expect(result.current.activeIndexes).toEqual([0, 1]) + expect(result.current.primaryIndex).toBe(0) + }) + + it('keeps the same gradient paint when an active word completes', () => { + const audio = createAudio({ currentTime: 0.25, paused: true }) + const { result } = renderTimeline({ audio }) + const tokenNode = registerToken( + result, + '0:stable-completion', + { start: 0, end: 500 }, + 'first', + ) + + const activeBackground = tokenNode.style.backgroundImage + expect(tokenNode.dataset.lyricsState).toBe('active') + expect(tokenNode.style.color).toBe('transparent') + + syncNow(result, 600) + + expect(tokenNode.dataset.lyricsState).toBe('completed') + expect(tokenNode.style.backgroundImage).toBe(activeBackground) + expect(tokenNode.style.color).toBe('transparent') + expect(tokenNode.style.webkitTextFillColor).toBe('transparent') + tokenNode + .querySelectorAll('[data-lyrics-character="true"]') + .forEach((character) => + expect(character.style.transform).toBe('translateY(-1.5000px)'), + ) + }) + + it('keeps gradient paint and opacity continuous when release becomes past', () => { + const audio = createAudio({ currentTime: 0.25, paused: true }) + const { result } = renderTimeline({ audio }) + const tokenNode = registerToken( + result, + '0:no-release-blink', + { start: 0, end: 500 }, + 'first', + ) + + syncNow(result, 1219) + const gradient = tokenNode.style.backgroundImage + const releaseAlpha = Number( + tokenNode.style.getPropertyValue('--lyrics-token-active-alpha'), + ) + expect(tokenNode.dataset.lyricsState).toBe('release') + expect(tokenNode.style.color).toBe('transparent') + expect(tokenNode.style.opacity).toBe('1') + + syncNow(result, 1220) + + const pastAlpha = Number( + tokenNode.style.getPropertyValue('--lyrics-token-active-alpha'), + ) + expect(tokenNode.dataset.lyricsState).toBe('inactive-past') + expect(tokenNode.style.backgroundImage).toBe(gradient) + expect(tokenNode.style.color).toBe('transparent') + expect(tokenNode.style.webkitTextFillColor).toBe('transparent') + expect(tokenNode.style.opacity).toBe('1') + expect(Math.abs(pastAlpha - releaseAlpha)).toBeLessThan(0.01) + expect(pastAlpha).toBeCloseTo(presentation.futureAlpha, 5) + expect(tokenNode.style.getPropertyValue('--lyrics-progress')).toBe('1') + }) + + it('uses the complete token duration for the character phase wave', () => { + const audio = createAudio({ currentTime: 0, duration: 5, paused: true }) + const longLines = [ + { + start: 0, + end: 4000, + tokens: [{ start: 0, end: 4000, value: 'super' }], + }, + ] + const { result } = renderTimeline({ audio, sourceLines: longLines }) + const tokenNode = registerToken( + result, + '0:long-word', + { start: 0, end: 4000 }, + 'super', + ) + + const characters = Array.from( + tokenNode.querySelectorAll('[data-lyrics-character="true"]'), + ) + expect(characters[0].style.backgroundImage).toBe( + tokenNode.style.backgroundImage, + ) + characters.forEach((character) => + expect(character.style.transform).not.toMatch(/^translateY\(\d/), + ) + const firstTransforms = [] + ;[0, 160, 320, 480].forEach((time) => { + syncNow(result, time) + firstTransforms.push(characters[0].style.transform) + }) + + expect(new Set(firstTransforms).size).toBe(firstTransforms.length) + firstTransforms.forEach((transform) => + expect(transform).toMatch(/^translateY\(-\d+\.\d{4}px\)$/), + ) + + syncNow(result, 1320) + const offsets = characters.map((character) => + Math.abs( + Number.parseFloat( + character.style.transform.match(/-?\d+\.\d+/)?.[0] || '0', + ), + ), + ) + offsets.forEach((offset) => expect(offset).toBeGreaterThanOrEqual(0)) + expect(offsets.filter((offset) => offset > 0)).toHaveLength(4) + expect(offsets[0]).toBeGreaterThan(offsets[1]) + expect(offsets[1]).toBeGreaterThan(offsets[2]) + expect(offsets[2]).toBeGreaterThan(offsets[3]) + expect(offsets[4]).toBe(0) + + syncNow(result, 3600) + expect(characters[4].style.transform).not.toBe( + `translateY(-${KARAOKE_CHARACTER_LIFT_PX.toFixed(4)}px)`, + ) + + syncNow(result, 3880) + characters.forEach((character) => + expect(character.style.transform).toBe( + `translateY(-${KARAOKE_CHARACTER_LIFT_PX.toFixed(4)}px)`, + ), + ) + }) + + it('keeps the normalized wave shape for short token durations', () => { + const audio = createAudio({ currentTime: 0, duration: 1, paused: true }) + const shortLines = [ + { + start: 0, + end: 180, + tokens: [{ start: 0, end: 180, value: 'go' }], + }, + ] + const { result } = renderTimeline({ audio, sourceLines: shortLines }) + const tokenNode = registerToken( + result, + '0:short-word', + { start: 0, end: 180 }, + 'go', + ) + syncNow(result, 0) + + const characters = tokenNode.querySelectorAll( + '[data-lyrics-character="true"]', + ) + expect(characters[0].style.transform).not.toBe('') + expect(characters[1].style.transform).not.toBe( + `translateY(-${KARAOKE_CHARACTER_LIFT_PX.toFixed(4)}px)`, + ) + + syncNow(result, 60) + expect(characters[1].style.transform).toBe( + `translateY(-${KARAOKE_CHARACTER_LIFT_PX.toFixed(4)}px)`, + ) + }) + + it('keeps interpolated playback time monotonic between coarse media updates', () => { + let frameCallback = null + let now = 0 + window.requestAnimationFrame.mockImplementation((callback) => { + frameCallback = callback + return 1 + }) + vi.spyOn(performance, 'now').mockImplementation(() => now) + + const audio = createAudio({ currentTime: 0.2, paused: false }) + const { result } = renderTimeline({ audio }) + const tokenNode = document.createElement('span') + act(() => { + result.current.registerToken( + '0:monotonic', + { + lineIndex: 0, + window: { start: 0, end: 1000 }, + presentation, + }, + tokenNode, + ) + }) + + now = 100 + act(() => frameCallback()) + const firstProgress = Number( + tokenNode.style.getPropertyValue('--lyrics-progress'), + ) + now = 180 + act(() => frameCallback()) + const secondProgress = Number( + tokenNode.style.getPropertyValue('--lyrics-progress'), + ) + + expect(secondProgress).toBeGreaterThanOrEqual(firstProgress) + }) + + it('resets monotonic interpolation after an explicit backward seek', () => { + let frameCallback = null + let now = 0 + window.requestAnimationFrame.mockImplementation((callback) => { + frameCallback = callback + return 1 + }) + vi.spyOn(performance, 'now').mockImplementation(() => now) + + const audio = createAudio({ currentTime: 0.8, paused: false }) + const { result } = renderTimeline({ audio }) + const tokenNode = registerToken(result, '0:backward-seek', { + start: 0, + end: 1000, + }) + + now = 100 + act(() => frameCallback()) + const beforeSeek = Number( + tokenNode.style.getPropertyValue('--lyrics-progress'), + ) + + audio.currentTime = 0.1 + act(() => { + audio.dispatchEvent(new Event('seeked')) + }) + now = 120 + act(() => frameCallback()) + + const afterSeek = Number( + tokenNode.style.getPropertyValue('--lyrics-progress'), + ) + expect(afterSeek).toBeLessThan(beforeSeek) + expect(afterSeek).toBeCloseTo(0.24, 2) + }) + + it('starts and stops requestAnimationFrame with playback visibility', () => { + const audio = createAudio({ currentTime: 0.25, paused: true }) + renderTimeline({ audio }) + + expect(window.requestAnimationFrame).not.toHaveBeenCalled() + + act(() => { + audio.paused = false + audio.dispatchEvent(new Event('play')) + }) + expect(window.requestAnimationFrame).toHaveBeenCalledTimes(1) + + act(() => { + audio.paused = true + audio.dispatchEvent(new Event('pause')) + }) + expect(window.cancelAnimationFrame).toHaveBeenCalled() + }) + + it('rebuilds open-line timing when media duration changes', () => { + const audio = createAudio({ duration: 10, paused: true }) + const { result } = renderTimeline({ + audio, + sourceLines: [{ start: 7000, tokens: [] }], + }) + + expect(result.current.timeline.windows[0].end).toBe(10000) + + act(() => { + audio.duration = 20 + audio.dispatchEvent(new Event('durationchange')) + }) + + expect(result.current.timeline.windows[0].end).toBe(15000) + }) +})