From b678aae393d4eca673a4f08efb7a634cdacd8602 Mon Sep 17 00:00:00 2001 From: Marcel van der Veldt Date: Fri, 28 Feb 2025 17:05:14 +0100 Subject: [PATCH] Fix Large PodcastEpisode listings not working (#1991) Convert Fix Large PodcastEpisode listings not working It turns out that some podcasts can contain many episodes, which could potentially result in a too large json message (especially with HA ingress). Adjust podcast episode listings to use asyncgenerator so it will be chunked sent over the api. --- music_assistant/controllers/media/podcasts.py | 26 +++++++++---------- music_assistant/controllers/player_queues.py | 5 +++- music_assistant/models/music_provider.py | 3 ++- .../providers/audiobookshelf/__init__.py | 8 +++--- .../providers/filesystem_local/__init__.py | 9 ++++--- .../providers/opensubsonic/sonic_provider.py | 12 +++------ .../providers/podcastfeed/__init__.py | 6 ++--- music_assistant/providers/test/__init__.py | 8 +++--- music_assistant/providers/ytmusic/__init__.py | 8 +++--- 9 files changed, 42 insertions(+), 43 deletions(-) diff --git a/music_assistant/controllers/media/podcasts.py b/music_assistant/controllers/media/podcasts.py index 6a6fd529..c7708536 100644 --- a/music_assistant/controllers/media/podcasts.py +++ b/music_assistant/controllers/media/podcasts.py @@ -2,7 +2,7 @@ from __future__ import annotations -import asyncio +from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any from music_assistant_models.enums import MediaType, ProviderFeature @@ -100,7 +100,7 @@ class PodcastsController(MediaControllerBase[Podcast]): self, item_id: str, provider_instance_id_or_domain: str, - ) -> UniqueList[PodcastEpisode]: + ) -> AsyncGenerator[PodcastEpisode, None]: """Return podcast episodes for the given provider podcast id.""" # always check if we have a library item for this podcast if provider_instance_id_or_domain == "library": @@ -113,10 +113,10 @@ class PodcastsController(MediaControllerBase[Podcast]): break # podcast episodes are not stored in the db/library # so we always need to fetch them from the provider - episodes = await self._get_provider_podcast_episodes( + async for episode in self._get_provider_podcast_episodes( item_id, provider_instance_id_or_domain - ) - return sorted(episodes, key=lambda x: x.position) + ): + yield episode async def episode( self, @@ -216,15 +216,11 @@ class PodcastsController(MediaControllerBase[Podcast]): async def _get_provider_podcast_episodes( self, item_id: str, provider_instance_id_or_domain: str - ) -> list[PodcastEpisode]: + ) -> AsyncGenerator[PodcastEpisode, None]: """Return podcast episodes for the given provider podcast id.""" prov: MusicProvider = self.mass.get_provider(provider_instance_id_or_domain) if prov is None: - return [] - # grab the episodes from the provider - # note that we do not cache any of this because its - # always a rather small list and we want fresh resume info - items = await prov.get_podcast_episodes(item_id) + return async def set_resume_position(episode: PodcastEpisode) -> None: if episode.fully_played is not None or episode.resume_position_ms: @@ -247,8 +243,12 @@ class PodcastsController(MediaControllerBase[Podcast]): if resume_info_db_row["fully_played"] is not None: episode.fully_played = resume_info_db_row["fully_played"] - await asyncio.gather(*[set_resume_position(x) for x in items]) - return items + # grab the episodes from the provider + # note that we do not cache any of this because its + # always a rather small list and we want fresh resume info + async for item in prov.get_podcast_episodes(item_id): + await set_resume_position(item) + yield item async def radio_mode_base_tracks( self, diff --git a/music_assistant/controllers/player_queues.py b/music_assistant/controllers/player_queues.py index facac860..923317d5 100644 --- a/music_assistant/controllers/player_queues.py +++ b/music_assistant/controllers/player_queues.py @@ -1338,7 +1338,10 @@ class PlayerQueuesController(CoreController): "Fetching episode(s) and resume point to play for Podcast %s", podcast.name, ) - all_episodes = await self.mass.music.podcasts.episodes(podcast.item_id, podcast.provider) + all_episodes = [ + x async for x in self.mass.music.podcasts.episodes(podcast.item_id, podcast.provider) + ] + all_episodes.sort(key=lambda x: x.position) # if a episode was provided, a user explicitly selected a episode to play # so we need to find the index of the episode in the list if isinstance(episode, PodcastEpisode): diff --git a/music_assistant/models/music_provider.py b/music_assistant/models/music_provider.py index a668b387..3c23c4aa 100644 --- a/music_assistant/models/music_provider.py +++ b/music_assistant/models/music_provider.py @@ -192,8 +192,9 @@ class MusicProvider(Provider): async def get_podcast_episodes( self, prov_podcast_id: str, - ) -> list[PodcastEpisode]: + ) -> AsyncGenerator[PodcastEpisode, None]: """Get all PodcastEpisodes for given podcast id.""" + yield if ProviderFeature.LIBRARY_PODCASTS in self.supported_features: raise NotImplementedError diff --git a/music_assistant/providers/audiobookshelf/__init__.py b/music_assistant/providers/audiobookshelf/__init__.py index 7058c00c..5b9cba5b 100644 --- a/music_assistant/providers/audiobookshelf/__init__.py +++ b/music_assistant/providers/audiobookshelf/__init__.py @@ -290,13 +290,14 @@ class Audiobookshelf(MusicProvider): base_url=str(self.config.get_value(CONF_URL)).rstrip("/"), ) - async def get_podcast_episodes(self, prov_podcast_id: str) -> list[PodcastEpisode]: + async def get_podcast_episodes( + self, prov_podcast_id: str + ) -> AsyncGenerator[PodcastEpisode, None]: """Get all podcast episodes of podcast. Adds progress information. """ abs_podcast = await self._get_abs_expanded_podcast(prov_podcast_id=prov_podcast_id) - episode_list = [] episode_cnt = 1 # the user has the progress of all media items # so we use a single api call here to obtain possibly many @@ -320,9 +321,8 @@ class Audiobookshelf(MusicProvider): base_url=str(self.config.get_value(CONF_URL)).rstrip("/"), media_progress=progress, ) - episode_list.append(mass_episode) + yield mass_episode episode_cnt += 1 - return episode_list async def get_podcast_episode( self, prov_episode_id: str, add_progress: bool = True diff --git a/music_assistant/providers/filesystem_local/__init__.py b/music_assistant/providers/filesystem_local/__init__.py index f46cc146..1b8b4533 100644 --- a/music_assistant/providers/filesystem_local/__init__.py +++ b/music_assistant/providers/filesystem_local/__init__.py @@ -637,7 +637,7 @@ class LocalFileSystemProvider(MusicProvider): async def get_podcast(self, prov_podcast_id: str) -> Podcast: """Get full podcast details by id.""" - for episode in await self.get_podcast_episodes(prov_podcast_id): + async for episode in self.get_podcast_episodes(prov_podcast_id): assert isinstance(episode.podcast, Podcast) return episode.podcast msg = f"Podcast not found: {prov_podcast_id}" @@ -701,7 +701,9 @@ class LocalFileSystemProvider(MusicProvider): ) return result - async def get_podcast_episodes(self, prov_podcast_id: str) -> list[PodcastEpisode]: + async def get_podcast_episodes( + self, prov_podcast_id: str + ) -> AsyncGenerator[PodcastEpisode, None]: """Get podcast episodes for given podcast id.""" episodes: list[PodcastEpisode] = [] @@ -726,7 +728,8 @@ class LocalFileSystemProvider(MusicProvider): continue tm.create_task(_process_podcast_episode(item)) - return episodes + for episode in episodes: + yield episode async def _parse_playlist_line(self, line: str, playlist_path: str) -> Track | None: """Try to parse a track from a playlist line.""" diff --git a/music_assistant/providers/opensubsonic/sonic_provider.py b/music_assistant/providers/opensubsonic/sonic_provider.py index ddfe8e7c..620ac827 100644 --- a/music_assistant/providers/opensubsonic/sonic_provider.py +++ b/music_assistant/providers/opensubsonic/sonic_provider.py @@ -607,7 +607,7 @@ class OpenSonicProvider(MusicProvider): async def get_podcast_episode(self, prov_episode_id: str) -> PodcastEpisode: """Get (full) podcast episode details by id.""" podcast_id, _ = prov_episode_id.split(EP_CHAN_SEP) - for episode in await self.get_podcast_episodes(podcast_id): + async for episode in self.get_podcast_episodes(podcast_id): if episode.item_id == prov_episode_id: return episode msg = f"Episode {prov_episode_id} not found" @@ -616,20 +616,16 @@ class OpenSonicProvider(MusicProvider): async def get_podcast_episodes( self, prov_podcast_id: str, - ) -> list[PodcastEpisode]: + ) -> AsyncGenerator[PodcastEpisode, None]: """Get all Episodes for given podcast id.""" if not self._enable_podcasts: - return [] - + return channels = await self._run_async( self._conn.getPodcasts, incEpisodes=True, pid=prov_podcast_id ) - channel = channels[0] - episodes = [] for episode in channel.episodes: - episodes.append(self._parse_epsiode(episode, channel)) - return episodes + yield self._parse_epsiode(episode, channel) async def get_podcast(self, prov_podcast_id: str) -> Podcast: """Get full Podcast details by id.""" diff --git a/music_assistant/providers/podcastfeed/__init__.py b/music_assistant/providers/podcastfeed/__init__.py index 83631705..c20f908d 100644 --- a/music_assistant/providers/podcastfeed/__init__.py +++ b/music_assistant/providers/podcastfeed/__init__.py @@ -157,14 +157,12 @@ class PodcastMusicprovider(MusicProvider): async def get_podcast_episodes( self, prov_podcast_id: str, - ) -> list[PodcastEpisode]: + ) -> AsyncGenerator[PodcastEpisode, None]: """List all episodes for the podcast.""" - episodes = [] if prov_podcast_id != self.podcast_id: raise Exception(f"Podcast id not in provider: {prov_podcast_id}") for idx, episode in enumerate(self.parsed["episodes"]): - episodes.append(await self._parse_episode(episode, idx)) - return episodes + yield await self._parse_episode(episode, idx) async def get_stream_details(self, item_id: str, media_type: MediaType) -> StreamDetails: """Get streamdetails for a track/radio.""" diff --git a/music_assistant/providers/test/__init__.py b/music_assistant/providers/test/__init__.py index d645ad52..a5687fb1 100644 --- a/music_assistant/providers/test/__init__.py +++ b/music_assistant/providers/test/__init__.py @@ -301,13 +301,11 @@ class TestProvider(MusicProvider): async def get_podcast_episodes( self, prov_podcast_id: str, - ) -> list[PodcastEpisode]: + ) -> AsyncGenerator[PodcastEpisode, None]: """Get all PodcastEpisodes for given podcast id.""" num_episodes = 25 - return [ - await self.get_podcast_episode(f"{prov_podcast_id}_{episode_idx}") - for episode_idx in range(num_episodes) - ] + for episode_idx in range(num_episodes): + yield await self.get_podcast_episode(f"{prov_podcast_id}_{episode_idx}") async def get_podcast_episode(self, prov_episode_id: str) -> PodcastEpisode: """Get (full) podcast episode details by id.""" diff --git a/music_assistant/providers/ytmusic/__init__.py b/music_assistant/providers/ytmusic/__init__.py index 4e39a0cf..adf286ee 100644 --- a/music_assistant/providers/ytmusic/__init__.py +++ b/music_assistant/providers/ytmusic/__init__.py @@ -408,18 +408,18 @@ class YoutubeMusicProvider(MusicProvider): podcast_obj = await get_podcast(prov_podcast_id, headers=self._headers) return self._parse_podcast(podcast_obj) - async def get_podcast_episodes(self, prov_podcast_id: str) -> list[PodcastEpisode]: + async def get_podcast_episodes( + self, prov_podcast_id: str + ) -> AsyncGenerator[PodcastEpisode, None]: """Get all episodes from a podcast.""" podcast_obj = await get_podcast(prov_podcast_id, headers=self._headers) podcast_obj["podcastId"] = prov_podcast_id podcast = self._parse_podcast(podcast_obj) - episodes = [] for index, episode_obj in enumerate(podcast_obj.get("episodes", []), start=1): episode = self._parse_podcast_episode(episode_obj, podcast) ep_index = episode_obj.get("index") or index episode.position = ep_index - episodes.append(episode) - return episodes + yield episode async def get_podcast_episode(self, prov_episode_id: str) -> PodcastEpisode: """Get a single Podcast Episode.""" -- 2.34.1