From fc5d0b9a099e4a153c77765004f4b18ad118b8fe Mon Sep 17 00:00:00 2001 From: Marcel van der Veldt Date: Mon, 29 Sep 2025 07:51:23 +0200 Subject: [PATCH] Several improvements to the Smartfades feature (#2433) --- music_assistant/controllers/music.py | 28 ++- music_assistant/controllers/player_queues.py | 41 +--- music_assistant/controllers/streams.py | 199 +++++++++++-------- music_assistant/helpers/audio.py | 29 ++- music_assistant/helpers/smart_fades.py | 155 +++++++++------ music_assistant/models/smart_fades.py | 10 +- music_assistant/providers/sonos/player.py | 2 + 7 files changed, 269 insertions(+), 195 deletions(-) diff --git a/music_assistant/controllers/music.py b/music_assistant/controllers/music.py index 68b7b27d..f7bd003a 100644 --- a/music_assistant/controllers/music.py +++ b/music_assistant/controllers/music.py @@ -73,7 +73,7 @@ from music_assistant.helpers.uri import parse_uri from music_assistant.helpers.util import TaskManager, parse_title_and_version from music_assistant.models.core_controller import CoreController from music_assistant.models.music_provider import MusicProvider -from music_assistant.models.smart_fades import SmartFadesAnalysis +from music_assistant.models.smart_fades import SmartFadesAnalysis, SmartFadesAnalysisFragment from .media.albums import AlbumsController from .media.artists import ArtistsController @@ -92,7 +92,7 @@ CONF_RESET_DB = "reset_db" DEFAULT_SYNC_INTERVAL = 12 * 60 # default sync interval in minutes CONF_SYNC_INTERVAL = "sync_interval" CONF_DELETED_PROVIDERS = "deleted_providers" -DB_SCHEMA_VERSION: Final[int] = 20 +DB_SCHEMA_VERSION: Final[int] = 21 CACHE_CATEGORY_LAST_SYNC: Final[int] = 9 @@ -850,6 +850,7 @@ class MusicController(CoreController): # skip invalid values return values = { + "fragment": analysis.fragment.value, "item_id": item_id, "provider": provider.lookup_key, "bpm": analysis.bpm, @@ -864,6 +865,7 @@ class MusicController(CoreController): self, item_id: str, provider_instance_id_or_domain: str, + fragment: SmartFadesAnalysisFragment, ) -> SmartFadesAnalysis | None: """Get Smart Fades BPM analysis for a track from db.""" if not (provider := self.mass.get_provider(provider_instance_id_or_domain)): @@ -873,10 +875,12 @@ class MusicController(CoreController): { "item_id": item_id, "provider": provider.lookup_key, + "fragment": fragment.value, }, ) if db_row and db_row["bpm"] > 0: return SmartFadesAnalysis( + fragment=SmartFadesAnalysisFragment(db_row["fragment"]), bpm=float(db_row["bpm"]), beats=np.array(json.loads(db_row["beats"])), downbeats=np.array(json.loads(db_row["downbeats"])), @@ -1725,9 +1729,18 @@ class MusicController(CoreController): if prev_version <= 20: # drop column cache_checksum from playlists table # this is no longer used and is a leftover from previous designs - await self.database.execute( - f"ALTER TABLE {DB_TABLE_PLAYLISTS} DROP COLUMN cache_checksum" - ) + try: + await self.database.execute( + f"ALTER TABLE {DB_TABLE_PLAYLISTS} DROP COLUMN cache_checksum" + ) + except Exception as err: + if "no such column" not in str(err): + raise + + if prev_version <= 21: + # drop table for smart fades analysis - it will be recreated with needed columns + await self.database.execute(f"DROP TABLE IF EXISTS {DB_TABLE_SMART_FADES_ANALYSIS}") + await self.__create_database_tables() # save changes await self.database.commit() @@ -1963,6 +1976,7 @@ class MusicController(CoreController): [id] INTEGER PRIMARY KEY AUTOINCREMENT, [item_id] TEXT NOT NULL, [provider] TEXT NOT NULL, + [fragment] INTEGER NOT NULL, [bpm] REAL NOT NULL, [beats] TEXT NOT NULL, [downbeats] TEXT NOT NULL, @@ -1970,7 +1984,7 @@ class MusicController(CoreController): [duration] REAL, [analysis_version] INTEGER DEFAULT 1, [timestamp_created] INTEGER DEFAULT (cast(strftime('%s','now') as int)), - UNIQUE(item_id,provider));""" + UNIQUE(item_id,provider,fragment));""" ) await self.database.commit() @@ -2081,7 +2095,7 @@ class MusicController(CoreController): # index on smart fades analysis table await self.database.execute( f"CREATE INDEX IF NOT EXISTS {DB_TABLE_SMART_FADES_ANALYSIS}_idx " - f"on {DB_TABLE_SMART_FADES_ANALYSIS}(item_id,provider);" + f"on {DB_TABLE_SMART_FADES_ANALYSIS}(item_id,provider,fragment);" ) await self.database.commit() diff --git a/music_assistant/controllers/player_queues.py b/music_assistant/controllers/player_queues.py index 2914787b..4ccc2014 100644 --- a/music_assistant/controllers/player_queues.py +++ b/music_assistant/controllers/player_queues.py @@ -65,7 +65,6 @@ from music_assistant.constants import ( ) from music_assistant.helpers.api import api_command from music_assistant.helpers.audio import get_stream_details, get_stream_dsp_details -from music_assistant.helpers.smart_fades import SmartFadesAnalyzer from music_assistant.helpers.throttle_retry import BYPASS_THROTTLER from music_assistant.helpers.util import get_changed_keys, percentage from music_assistant.models.core_controller import CoreController @@ -142,7 +141,6 @@ class PlayerQueuesController(CoreController): "Music Assistant's core controller which manages the queues for all players." ) self.manifest.icon = "playlist-music" - self._smart_fades_analyzer = SmartFadesAnalyzer(self.mass) async def close(self) -> None: """Cleanup on exit.""" @@ -1009,13 +1007,13 @@ class PlayerQueuesController(CoreController): self._queues.pop(player_id, None) self._queue_items.pop(player_id, None) - async def preload_next_queue_item( + async def load_next_queue_item( self, queue_id: str, current_item_id: str, ) -> QueueItem: """ - Call when a player wants (to preload) the next queue item to play. + Call when a player wants the next queue item to play. Raises QueueEmpty if there are no more tracks left. """ @@ -1161,9 +1159,9 @@ class PlayerQueuesController(CoreController): ) # allow stripping silence from the begin/end of the track if crossfade is enabled # this will allow for (much) smoother crossfades - if await self.mass.config.get_player_config_value(queue_id, CONF_SMART_FADES_MODE) in ( - SmartFadesMode.STANDARD_CROSSFADE, - SmartFadesMode.SMART_FADES, + if ( + await self.mass.config.get_player_config_value(queue_id, CONF_SMART_FADES_MODE) + != SmartFadesMode.DISABLED ): queue_item.streamdetails.strip_silence_end = True queue_item.streamdetails.strip_silence_begin = not is_start @@ -1591,15 +1589,9 @@ class PlayerQueuesController(CoreController): retries -= 1 await asyncio.sleep(1) - if next_item := await self.preload_next_queue_item(queue_id, item_id_in_buffer): + if next_item := await self.load_next_queue_item(queue_id, item_id_in_buffer): self._enqueue_next_item(queue_id, next_item) - if ( - await self.mass.config.get_player_config_value( - queue_id, CONF_SMART_FADES_MODE - ) - == SmartFadesMode.SMART_FADES - ): - self._trigger_smart_fades_analysis(next_item) + except QueueEmpty: return @@ -2139,22 +2131,3 @@ class PlayerQueuesController(CoreController): is_playing=is_playing, ), ) - - def _trigger_smart_fades_analysis(self, next_item: QueueItem) -> None: - """Trigger analysis for smart fades if needed.""" - if not next_item.streamdetails: - self.logger.warning("No stream details for smart fades analysis: %s", next_item.name) - return - if next_item.streamdetails.smart_fades: - return - - async def _trigger_smart_fades_analysis(next_item: QueueItem) -> None: - analysis = await self._smart_fades_analyzer.analyze(next_item.streamdetails) - # Store the analysis on the queue item for future reference - next_item.streamdetails.smart_fades = analysis - - task_id = ( - f"smart_fades_analysis_{next_item.streamdetails.provider}_" - f"{next_item.streamdetails.item_id}" - ) - self.mass.create_task(_trigger_smart_fades_analysis, next_item, task_id=task_id) diff --git a/music_assistant/controllers/streams.py b/music_assistant/controllers/streams.py index 0a581274..f55fb34f 100644 --- a/music_assistant/controllers/streams.py +++ b/music_assistant/controllers/streams.py @@ -13,7 +13,7 @@ import logging import os import urllib.parse from collections.abc import AsyncGenerator -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING, TypedDict from aiofiles.os import wrap @@ -61,12 +61,13 @@ from music_assistant.helpers.audio import ( get_player_filter_params, get_silence, get_stream_details, + resample_pcm_audio, ) from music_assistant.helpers.audio import LOGGER as AUDIO_LOGGER from music_assistant.helpers.ffmpeg import LOGGER as FFMPEG_LOGGER from music_assistant.helpers.ffmpeg import check_ffmpeg_version, get_ffmpeg_stream from music_assistant.helpers.smart_fades import ( - MAX_SMART_CROSSFADE_DURATION, + SMART_CROSSFADE_DURATION, SmartFadesMixer, SmartFadesMode, ) @@ -80,12 +81,12 @@ from music_assistant.helpers.util import ( from music_assistant.helpers.webserver import Webserver from music_assistant.models.core_controller import CoreController from music_assistant.models.plugin import PluginProvider -from music_assistant.models.smart_fades import SmartFadesAnalysis if TYPE_CHECKING: from music_assistant_models.config_entries import CoreConfig from music_assistant_models.player_queue import PlayerQueue from music_assistant_models.queue_item import QueueItem + from music_assistant_models.streamdetails import StreamDetails from music_assistant.mass import MusicAssistant from music_assistant.models.player import Player @@ -109,11 +110,11 @@ def parse_pcm_info(content_type: str) -> tuple[int, int, int]: class CrossfadeData: """Data class to hold crossfade data.""" - fadeout_part: bytes = b"" - pcm_format: AudioFormat = field(default_factory=AudioFormat) - queue_item_id: str | None = None - session_id: str | None = None - smart_fades_analysis: SmartFadesAnalysis | None = None + data: bytes + fade_in_size: int + pcm_format: AudioFormat + queue_item_id: str + session_id: str class AnnounceData(TypedDict): @@ -812,10 +813,9 @@ class StreamsController(CoreController): # ruff: noqa: PLR0915 assert pcm_format.content_type.is_pcm() queue_track = None - last_fadeout_part = b"" - last_fadeout_analysis = ( - None # Smart fades analysis for the track that created last_fadeout_part - ) + last_fadeout_part: bytes = b"" + last_streamdetails: StreamDetails | None = None + last_play_log_entry: PlayLogEntry | None = None queue.flow_mode = True if not start_queue_item: # this can happen in some (edge case) race conditions @@ -845,7 +845,7 @@ class StreamsController(CoreController): queue_track = start_queue_item else: try: - queue_track = await self.mass.player_queues.preload_next_queue_item( + queue_track = await self.mass.player_queues.load_next_queue_item( queue.queue_id, queue_track.queue_item_id ) except QueueEmpty: @@ -866,7 +866,10 @@ class StreamsController(CoreController): # append to play log so the queue controller can work out which track is playing play_log_entry = PlayLogEntry(queue_track.queue_item_id) queue.flow_mode_stream_log.append(play_log_entry) - crossfade_size = int(pcm_format.pcm_sample_size * MAX_SMART_CROSSFADE_DURATION) + if smart_fades_mode == SmartFadesMode.SMART_FADES: + crossfade_size = int(pcm_format.pcm_sample_size * SMART_CROSSFADE_DURATION) + else: + crossfade_size = int(pcm_format.pcm_sample_size * standard_crossfade_duration + 4) bytes_written = 0 buffer = b"" # handle incoming audio chunks @@ -889,26 +892,34 @@ class StreamsController(CoreController): continue #### HANDLE CROSSFADE OF PREVIOUS TRACK AND NEW TRACK - if last_fadeout_part: + if last_fadeout_part and last_streamdetails: # perform crossfade fadein_part = buffer[:crossfade_size] remaining_bytes = buffer[crossfade_size:] - # Use the mixer to handle all crossfade logic crossfade_part = await self._smart_fades_mixer.mix( fade_in_part=fadein_part, fade_out_part=last_fadeout_part, - fade_in_analysis=queue_track.streamdetails.smart_fades, - fade_out_analysis=last_fadeout_analysis, + fade_in_streamdetails=queue_track.streamdetails, + fade_out_streamdetails=last_streamdetails, pcm_format=pcm_format, standard_crossfade_duration=standard_crossfade_duration, mode=smart_fades_mode, ) - + # because the crossfade exists of both the fadein and fadeout part + # we need to correct the bytes_written accordingly so the duration + # calculations at the end of the track are correct + crossfade_part_len = len(crossfade_part) + bytes_written += crossfade_part_len / 2 + if last_play_log_entry: + last_play_log_entry.seconds_streamed += ( + crossfade_part_len / 2 / pcm_sample_size + ) # send crossfade_part (as one big chunk) - bytes_written += len(crossfade_part) yield crossfade_part + del crossfade_part + # also write the leftover bytes from the crossfade action if remaining_bytes: yield remaining_bytes @@ -916,7 +927,7 @@ class StreamsController(CoreController): del remaining_bytes # clear vars last_fadeout_part = b"" - last_fadeout_analysis = None + last_streamdetails = None buffer = b"" #### OTHER: enough data in buffer, feed to output @@ -931,12 +942,11 @@ class StreamsController(CoreController): yield last_fadeout_part bytes_written += len(last_fadeout_part) last_fadeout_part = b"" - last_fadeout_analysis = None if self._crossfade_allowed(queue_track, flow_mode=True): # if crossfade is enabled, save fadeout part to pickup for next track last_fadeout_part = buffer[-crossfade_size:] - # Also save the smart fades analysis - last_fadeout_analysis = queue_track.streamdetails.smart_fades + last_streamdetails = queue_track.streamdetails + last_play_log_entry = play_log_entry remaining_bytes = buffer[:-crossfade_size] if remaining_bytes: yield remaining_bytes @@ -973,7 +983,7 @@ class StreamsController(CoreController): last_part_seconds = len(last_fadeout_part) / pcm_sample_size queue_track.streamdetails.seconds_streamed += last_part_seconds queue_track.streamdetails.duration += last_part_seconds - del last_fadeout_part + last_fadeout_part = b"" total_bytes_sent += bytes_written self.logger.info("Finished Queue Flow stream for Queue %s", queue.display_name) @@ -1144,25 +1154,27 @@ class StreamsController(CoreController): streamdetails = queue_item.streamdetails assert streamdetails - self._crossfade_data.setdefault(queue.queue_id, CrossfadeData()) - crossfade_data = self._crossfade_data[queue.queue_id] + crossfade_data = self._crossfade_data.get(queue.queue_id) self.logger.debug( "Start Streaming queue track: %s (%s) for queue %s - crossfade: %s", queue_item.streamdetails.uri if queue_item.streamdetails else "Unknown URI", queue_item.name, queue.display_name, - f"{standard_crossfade_duration} seconds", + smart_fades_mode, ) - if crossfade_data.session_id != session_id: + if crossfade_data and crossfade_data.session_id != session_id: # invalidate expired crossfade data - crossfade_data.fadeout_part = b"" - crossfade_data.smart_fades_analysis = None + crossfade_data = None buffer = b"" bytes_written = 0 - crossfade_size = int(pcm_format.pcm_sample_size * MAX_SMART_CROSSFADE_DURATION) + if smart_fades_mode == SmartFadesMode.SMART_FADES: + crossfade_size = int(pcm_format.pcm_sample_size * SMART_CROSSFADE_DURATION) + else: + crossfade_size = int(pcm_format.pcm_sample_size * standard_crossfade_duration + 4) + fade_out_data: bytes | None = None async for chunk in self.get_queue_item_stream(queue_item, pcm_format): # ALWAYS APPEND CHUNK TO BUFFER @@ -1172,36 +1184,24 @@ class StreamsController(CoreController): # buffer is not full enough, move on continue - #### HANDLE CROSSFADE OF PREVIOUS TRACK AND NEW TRACK - if crossfade_data and crossfade_data.fadeout_part: - # perform crossfade - fade_in_part = buffer[:crossfade_size] - remaining_bytes = buffer[crossfade_size:] - - # Check if both tracks have smart fades analysis for BPM matching - crossfade_part = await self._smart_fades_mixer.mix( - fade_in_part=fade_in_part, - fade_out_part=crossfade_data.fadeout_part, - fade_in_analysis=queue_item.streamdetails.smart_fades, - fade_out_analysis=crossfade_data.smart_fades_analysis, - pcm_format=pcm_format, - standard_crossfade_duration=standard_crossfade_duration, - mode=smart_fades_mode, - ) - # send crossfade_part (as one big chunk) - bytes_written += len(crossfade_part) - yield crossfade_part - - # also write the leftover bytes from the crossfade action - if remaining_bytes: - yield remaining_bytes - bytes_written += len(remaining_bytes) - del remaining_bytes + #### HANDLE CROSSFADE DATA FROM PREVIOUS TRACK + if crossfade_data: + # discard the fade_in_part from the crossfade data + buffer = buffer[crossfade_data.fade_in_size :] + # send the (second half of the) crossfade data + if crossfade_data.pcm_format != pcm_format: + # pcm format mismatch, we need to resample the crossfade data + async for _crossfade_chunk in resample_pcm_audio( + crossfade_data.data, crossfade_data.pcm_format, pcm_format + ): + yield _crossfade_chunk + bytes_written += len(_crossfade_chunk) + del _crossfade_chunk + else: + yield crossfade_data.data + bytes_written += len(crossfade_data.data) # clear vars - crossfade_data.fadeout_part = b"" - crossfade_data.smart_fades_analysis = None - buffer = b"" - del fade_in_part + crossfade_data = None #### OTHER: enough data in buffer, feed to output while len(buffer) > crossfade_size: @@ -1210,42 +1210,73 @@ class StreamsController(CoreController): buffer = buffer[pcm_format.pcm_sample_size :] #### HANDLE END OF TRACK - if crossfade_data and crossfade_data.fadeout_part: - # edge case: we did not get enough data to make the crossfade - if crossfade_data.pcm_format == pcm_format: - yield crossfade_data.fadeout_part - bytes_written += len(crossfade_data.fadeout_part) - # always reset fadeout part at this point - crossfade_data.fadeout_part = b"" - crossfade_data.smart_fades_analysis = None - if self._crossfade_allowed(queue_item, flow_mode=False): + + if not self._crossfade_allowed(queue_item, flow_mode=False): + # no crossfade enabled/allowed, just yield the buffer last part + bytes_written += len(buffer) + yield buffer + else: # if crossfade is enabled, save fadeout part to pickup for next track - crossfade_data.fadeout_part = buffer[-crossfade_size:] - crossfade_data.pcm_format = pcm_format - crossfade_data.session_id = session_id - crossfade_data.queue_item_id = queue_item.queue_item_id - # Also save the smart fades analysis for BPM matching - crossfade_data.smart_fades_analysis = queue_item.streamdetails.smart_fades + fade_out_data = buffer[-crossfade_size:] remaining_bytes = buffer[:-crossfade_size] if remaining_bytes: yield remaining_bytes bytes_written += len(remaining_bytes) del remaining_bytes - elif buffer: - # no crossfade enabled/allowed, just yield the buffer last part - bytes_written += len(buffer) - yield buffer + buffer = b"" + # get next track for crossfade + try: + next_queue_item = await self.mass.player_queues.load_next_queue_item( + queue.queue_id, queue_item.queue_item_id + ) + async for chunk in self.get_queue_item_stream(next_queue_item, pcm_format): + # ALWAYS APPEND CHUNK TO BUFFER + buffer += chunk + del chunk + if len(buffer) < crossfade_size: + # buffer is not full enough, move on + continue + #### HANDLE CROSSFADE OF PREVIOUS TRACK AND NEW TRACK + crossfade_data = await self._smart_fades_mixer.mix( + fade_in_part=buffer, + fade_out_part=fade_out_data, + fade_in_streamdetails=next_queue_item.streamdetails, + fade_out_streamdetails=queue_item.streamdetails, + pcm_format=pcm_format, + standard_crossfade_duration=standard_crossfade_duration, + mode=smart_fades_mode, + ) + # send half of the crossfade_part (= approx the fadeout part) + crossfade_first, crossfade_second = ( + crossfade_data[: len(crossfade_data) // 2 + len(crossfade_data) % 2], + crossfade_data[len(crossfade_data) // 2 + len(crossfade_data) % 2 :], + ) + bytes_written += len(crossfade_first) + yield crossfade_first + del crossfade_first + # store the other half for the next track + self._crossfade_data[queue_item.queue_id] = CrossfadeData( + data=crossfade_second, + fade_in_size=len(buffer), + pcm_format=pcm_format, + queue_item_id=next_queue_item.queue_item_id, + session_id=session_id, + ) + # clear vars and break out of loop + del crossfade_data + break + except QueueEmpty: + # end of queue reached or crossfade failed - no crossfade possible + yield fade_out_data + bytes_written += len(fade_out_data) + del fade_out_data # make sure the buffer gets cleaned up del buffer - # update duration details based on the actual pcm data we sent # this also accounts for crossfade and silence stripping seconds_streamed = bytes_written / pcm_format.pcm_sample_size streamdetails.seconds_streamed = seconds_streamed streamdetails.duration = streamdetails.seek_position + seconds_streamed - queue_item.duration = streamdetails.duration - if queue_item.media_type: - queue_item.media_item.duration = streamdetails.duration self.logger.debug( "Finished Streaming queue track: %s (%s) on queue %s", queue_item.streamdetails.uri, diff --git a/music_assistant/helpers/audio.py b/music_assistant/helpers/audio.py index a346902e..bd03bb39 100644 --- a/music_assistant/helpers/audio.py +++ b/music_assistant/helpers/audio.py @@ -627,15 +627,6 @@ async def get_stream_details( streamdetails.loudness = result[0] streamdetails.loudness_album = result[1] streamdetails.prefer_album_loudness = prefer_album_loudness - - # handle smart fades analysis details - if queue_item.media_type == MediaType.TRACK: - if smart_fades_analysis := await mass.music.get_smart_fades_analysis( - streamdetails.item_id, - streamdetails.provider, - ): - LOGGER.debug("Found smart fades analysis in the database for %s", queue_item.uri) - streamdetails.smart_fades = smart_fades_analysis player_settings = await mass.config.get_player_config(streamdetails.queue_id) core_config = await mass.config.get_core_config("streams") streamdetails.target_loudness = float( @@ -1422,6 +1413,26 @@ async def get_silence( yield chunk +async def resample_pcm_audio( + input_audio: bytes | AsyncGenerator[bytes, None], + input_format: AudioFormat, + output_format: AudioFormat, +) -> AsyncGenerator[bytes, None]: + """Resample (a chunk of) PCM audio from input_format to output_format using ffmpeg.""" + LOGGER.debug(f"Resampling audio from {input_format} to {output_format}") + + async def _yielder() -> AsyncGenerator[bytes, None]: + yield input_audio # type: ignore[misc] + + async for chunk in get_ffmpeg_stream( + audio_input=_yielder() if isinstance(input_audio, bytes) else input_audio, + input_format=input_format, + output_format=output_format, + raise_ffmpeg_exception=True, + ): + yield chunk + + def get_chunksize( fmt: AudioFormat, seconds: int = 1, diff --git a/music_assistant/helpers/smart_fades.py b/music_assistant/helpers/smart_fades.py index 69db9cfd..32b12e97 100644 --- a/music_assistant/helpers/smart_fades.py +++ b/music_assistant/helpers/smart_fades.py @@ -9,35 +9,32 @@ from __future__ import annotations import asyncio import logging import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import aiofiles import librosa import numpy as np import numpy.typing as npt import shortuuid -from music_assistant_models.enums import ContentType, MediaType -from music_assistant_models.media_items import AudioFormat from music_assistant.constants import VERBOSE_LOG_LEVEL -from music_assistant.helpers.audio import crossfade_pcm_parts, get_media_stream +from music_assistant.helpers.audio import crossfade_pcm_parts from music_assistant.helpers.process import communicate from music_assistant.helpers.util import remove_file from music_assistant.models.smart_fades import ( SmartFadesAnalysis, + SmartFadesAnalysisFragment, SmartFadesMode, ) if TYPE_CHECKING: + from music_assistant_models.media_items import AudioFormat from music_assistant_models.streamdetails import StreamDetails from music_assistant.mass import MusicAssistant -MAX_SMART_CROSSFADE_DURATION = 45 +SMART_CROSSFADE_DURATION = 45 ANALYSIS_FPS = 100 -ANALYSIS_PCM_FORMAT = AudioFormat( - content_type=ContentType.PCM_F32LE, sample_rate=44100, bit_depth=32, channels=1 -) # Only apply time stretching if BPM difference is < this % TIME_STRETCH_BPM_PERCENTAGE_THRESHOLD = 8.0 @@ -52,28 +49,40 @@ class SmartFadesAnalyzer: async def analyze( self, - streamdetails: StreamDetails, + item_id: str, + provider_instance_id_or_domain: str, + fragment: SmartFadesAnalysisFragment, + audio_data: bytes, + pcm_format: AudioFormat, ) -> SmartFadesAnalysis | None: """Analyze a track's beats for BPM matching smart fade.""" - stream_details_name = f"{streamdetails.provider}://{streamdetails.item_id}" - if streamdetails.media_type != MediaType.TRACK: - self.logger.debug( - "Skipping smart fades analysis for non-track item: %s", stream_details_name - ) - return None - + stream_details_name = f"{provider_instance_id_or_domain}://{item_id}" start_time = time.perf_counter() - self.logger.debug("Starting beat analysis for track : %s", stream_details_name) + self.logger.debug( + "Starting %s beat analysis for track : %s", fragment.name, stream_details_name + ) + fragment_duration = len(audio_data) / (pcm_format.pcm_sample_size) try: - audio_data = await self._get_audio_bytes_from_stream_details(streamdetails) self.logger.log( VERBOSE_LOG_LEVEL, "Audio data: %.2fs, %d bytes", - streamdetails.duration or 0, + fragment_duration, len(audio_data), ) # Perform beat analysis - analysis = await self._analyze_track_beats(audio_data) + + # Convert PCM bytes to numpy array and then to mono for analysis + audio_array = np.frombuffer(audio_data, dtype=np.float32) + if pcm_format.channels > 1: + # Reshape to separate channels and take average for mono conversion + audio_array = audio_array.reshape(-1, pcm_format.channels) + mono_audio = np.asarray(np.mean(audio_array, axis=1, dtype=np.float32)) + else: + # Single channel - ensure consistent array type + mono_audio = np.asarray(audio_array, dtype=np.float32) + + analysis = await self._analyze_track_beats(mono_audio, fragment, pcm_format.sample_rate) + total_time = time.perf_counter() - start_time if not analysis: self.logger.debug( @@ -94,7 +103,7 @@ class SmartFadesAnalyzer: ) self.mass.create_task( self.mass.music.set_smart_fades_analysis( - streamdetails.item_id, streamdetails.provider, analysis + item_id, provider_instance_id_or_domain, analysis ) ) return analysis @@ -109,15 +118,20 @@ class SmartFadesAnalyzer: return None def _librosa_beat_analysis( - self, audio_array: npt.NDArray[np.float32] + self, + audio_array: npt.NDArray[np.float32], + fragment: SmartFadesAnalysisFragment, + sample_rate: int, ) -> SmartFadesAnalysis | None: """Perform beat analysis using librosa.""" try: tempo, beats_array = librosa.beat.beat_track( y=audio_array, - sr=ANALYSIS_PCM_FORMAT.sample_rate, + sr=sample_rate, units="time", ) + # librosa returns np.float64 arrays when units="time" + if len(beats_array) < 2: self.logger.warning("Insufficient beats detected: %d", len(beats_array)) return None @@ -137,15 +151,16 @@ class SmartFadesAnalyzer: downbeats = self._estimate_musical_downbeats(beats_array, bpm) - # Store complete track analysis - track_duration = len(audio_array) / ANALYSIS_PCM_FORMAT.sample_rate + # Store complete fragment analysis + fragment_duration = len(audio_array) / sample_rate return SmartFadesAnalysis( + fragment=fragment, bpm=float(bpm), beats=beats_array, downbeats=downbeats, confidence=float(confidence), - duration=track_duration, + duration=fragment_duration, ) except Exception as e: @@ -201,33 +216,17 @@ class SmartFadesAnalyzer: return downbeats - async def _get_audio_bytes_from_stream_details(self, streamdetails: StreamDetails) -> bytes: - """Retrieve bytes from the audio stream.""" - audio_data = bytearray() - async for chunk in get_media_stream( - self.mass, - streamdetails=streamdetails, - pcm_format=ANALYSIS_PCM_FORMAT, - filter_params=[], - ): - audio_data.extend(chunk) - if not audio_data: - self.logger.warning( - "No audio data received for analysis: %s", - f"{streamdetails.provider}/{streamdetails.item_id}", - ) - return b"" - return bytes(audio_data) - async def _analyze_track_beats( self, - audio_data: bytes, + audio_data: npt.NDArray[np.float32], + fragment: SmartFadesAnalysisFragment, + sample_rate: int, ) -> SmartFadesAnalysis | None: """Analyze track for beat tracking using librosa.""" try: - # Convert PCM bytes directly to numpy array (mono audio) - audio_array = np.frombuffer(audio_data, dtype=np.float32) - return await asyncio.to_thread(self._librosa_beat_analysis, audio_array) + return await asyncio.to_thread( + self._librosa_beat_analysis, audio_data, fragment, sample_rate + ) except Exception as e: self.logger.exception("Beat tracking analysis failed: %s", e) return None @@ -240,18 +239,51 @@ class SmartFadesMixer: """Initialize smart fades mixer.""" self.mass = mass self.logger = logging.getLogger(__name__) + # TODO: Refactor into stream (or metadata) controller after we have split the controllers + self.analyzer = SmartFadesAnalyzer(mass) async def mix( self, fade_in_part: bytes, fade_out_part: bytes, - fade_in_analysis: SmartFadesAnalysis, - fade_out_analysis: SmartFadesAnalysis, + fade_in_streamdetails: StreamDetails, + fade_out_streamdetails: StreamDetails, pcm_format: AudioFormat, standard_crossfade_duration: int = 10, mode: SmartFadesMode = SmartFadesMode.SMART_FADES, ) -> bytes: """Apply crossfade with internal state management and smart/standard fallback logic.""" + fade_out_analysis: SmartFadesAnalysis | None + if stored_analysis := await self.mass.music.get_smart_fades_analysis( + fade_out_streamdetails.item_id, + fade_out_streamdetails.provider, + SmartFadesAnalysisFragment.OUTRO, + ): + fade_out_analysis = stored_analysis + else: + fade_out_analysis = await self.analyzer.analyze( + fade_out_streamdetails.item_id, + fade_out_streamdetails.provider, + SmartFadesAnalysisFragment.OUTRO, + fade_out_part, + pcm_format, + ) + + fade_in_analysis: SmartFadesAnalysis | None + if stored_analysis := await self.mass.music.get_smart_fades_analysis( + fade_in_streamdetails.item_id, + fade_in_streamdetails.provider, + SmartFadesAnalysisFragment.INTRO, + ): + fade_in_analysis = stored_analysis + else: + fade_in_analysis = await self.analyzer.analyze( + fade_in_streamdetails.item_id, + fade_in_streamdetails.provider, + SmartFadesAnalysisFragment.INTRO, + fade_in_part, + pcm_format, + ) if ( fade_out_analysis and fade_in_analysis @@ -348,7 +380,7 @@ class SmartFadesMixer: ] ) - self.logger.debug("FFmpeg command args: %s", " ".join(args)) + self.logger.log(VERBOSE_LOG_LEVEL, "FFmpeg command args: %s", " ".join(args)) # Execute the enhanced smart fade with full buffer _, raw_crossfade_output, stderr = await communicate(args, fade_in_part) @@ -401,13 +433,13 @@ class SmartFadesMixer: # Check if we would have enough audio after beat alignment for the crossfade if ( fadein_start_pos is not None - and fadein_start_pos + crossfade_duration > MAX_SMART_CROSSFADE_DURATION + and fadein_start_pos + crossfade_duration > SMART_CROSSFADE_DURATION ): self.logger.debug( "Skipping beat alignment: not enough audio after trim (%.1fs + %.1fs > %.1fs)", fadein_start_pos, crossfade_duration, - MAX_SMART_CROSSFADE_DURATION, + SMART_CROSSFADE_DURATION, ) # Skip beat alignment fadein_start_pos = None @@ -455,10 +487,10 @@ class SmartFadesMixer: musical_duration = crossfade_bars * beats_per_bar * seconds_per_beat # Apply buffer constraint - actual_duration = min(musical_duration, MAX_SMART_CROSSFADE_DURATION) + actual_duration = min(musical_duration, SMART_CROSSFADE_DURATION) # Log if we had to constrain the duration - if musical_duration > MAX_SMART_CROSSFADE_DURATION: + if musical_duration > SMART_CROSSFADE_DURATION: self.logger.debug( "Constraining crossfade duration from %.1fs to %.1fs (buffer limit)", musical_duration, @@ -503,7 +535,7 @@ class SmartFadesMixer: ) # Check if it fits in fadein buffer - fadein_buffer = MAX_SMART_CROSSFADE_DURATION - fadein_start_pos + fadein_buffer = SMART_CROSSFADE_DURATION - fadein_start_pos if test_duration <= fadein_buffer: if bars < ideal_bars: self.logger.debug( @@ -529,7 +561,9 @@ class SmartFadesMixer: # Helper function to calculate beat positions from beat arrays def calculate_beat_positions( - fade_out_beats: Any, fade_in_beats: Any, num_beats: int + fade_out_beats: npt.NDArray[np.float64], + fade_in_beats: npt.NDArray[np.float64], + num_beats: int, ) -> tuple[float, float] | None: """Calculate start positions from beat arrays with phantom downbeat support.""" if len(fade_out_beats) < num_beats or len(fade_in_beats) < num_beats: @@ -695,7 +729,7 @@ class SmartFadesMixer: # Calculate the tempo change factor # atempo accepts values between 0.5 and 2.0 (can be chained for larger changes) tempo_factor = bpm_ratio - buffer_duration = MAX_SMART_CROSSFADE_DURATION # 45 seconds + buffer_duration = SMART_CROSSFADE_DURATION # 45 seconds # Calculate expected crossfade duration from bars for comparison beats_per_bar = 4 @@ -807,14 +841,14 @@ class SmartFadesMixer: crossover_freq = int(crossover_freq * 0.85) # Extended lowpass effect to gradually remove bass frequencies - fadeout_eq_duration = min(max(crossfade_duration * 2.5, 8.0), MAX_SMART_CROSSFADE_DURATION) + fadeout_eq_duration = min(max(crossfade_duration * 2.5, 8.0), SMART_CROSSFADE_DURATION) # Quicker highpass removal to avoid lingering vocals after crossfade fadein_eq_duration = crossfade_duration / 1.5 # Calculate when the EQ sweep should start # The crossfade always happens at the END of the buffer, regardless of beat alignment - fadeout_eq_start = max(0, MAX_SMART_CROSSFADE_DURATION - fadeout_eq_duration) + fadeout_eq_start = max(0, SMART_CROSSFADE_DURATION - fadeout_eq_duration) self.logger.debug( "EQ: crossover=%dHz, EQ fadeout duration=%.1fs" @@ -872,6 +906,7 @@ class SmartFadesMixer: fade_in_part[:crossfade_size], fade_out_part[-crossfade_size:], pcm_format=pcm_format, + fade_out_pcm_format=pcm_format, ) # Post-crossfade: incoming track minus the crossfaded portion post_crossfade = fade_in_part[crossfade_size:] diff --git a/music_assistant/models/smart_fades.py b/music_assistant/models/smart_fades.py index 64ebe97f..63bd155e 100644 --- a/music_assistant/models/smart_fades.py +++ b/music_assistant/models/smart_fades.py @@ -1,7 +1,7 @@ """Data models for Smart Fades analysis and configuration.""" from dataclasses import dataclass -from enum import StrEnum +from enum import IntEnum, StrEnum import numpy as np import numpy.typing as npt @@ -17,10 +17,18 @@ class SmartFadesMode(StrEnum): DISABLED = "disabled" # No crossfade +class SmartFadesAnalysisFragment(IntEnum): + """Smart fades analysis fragment types.""" + + INTRO = 1 + OUTRO = 2 + + @dataclass class SmartFadesAnalysis(DataClassDictMixin): """Beat tracking analysis data for BPM matching crossfade.""" + fragment: SmartFadesAnalysisFragment bpm: float beats: npt.NDArray[np.float64] # Beat positions downbeats: npt.NDArray[np.float64] # Downbeat positions diff --git a/music_assistant/providers/sonos/player.py b/music_assistant/providers/sonos/player.py index bdaa577e..9dcc7fe0 100644 --- a/music_assistant/providers/sonos/player.py +++ b/music_assistant/providers/sonos/player.py @@ -66,6 +66,8 @@ SUPPORTED_FEATURES = { PlayerFeature.SELECT_SOURCE, PlayerFeature.ENQUEUE, PlayerFeature.SET_MEMBERS, + PlayerFeature.GAPLESS_PLAYBACK, + PlayerFeature.GAPLESS_DIFFERENT_SAMPLERATE, } -- 2.34.1