From e973303cfc4e3ddb72f2fe722c1a9c0ac5a6e77b Mon Sep 17 00:00:00 2001 From: Jozef Kruszynski <60214390+jozefKruszynski@users.noreply.github.com> Date: Fri, 26 May 2023 14:43:55 +0200 Subject: [PATCH] Add availability to tidal album provider mapping (#664) * Add availability to tidal album provider mapping * Remove extraneous catch and re-throw * Hopefully this is more sane * Fix overly long log lines * Fixed impossible returns, added rate limiting * Fix ruff lint error * Fixed throttler nonsense * Stop swallowing errors * Cleanup last few suggestions --------- Co-authored-by: jkruszynski --- .../server/controllers/media/albums.py | 2 + .../server/providers/tidal/__init__.py | 115 +++++++++--------- .../server/providers/tidal/helpers.py | 83 +++++++++---- 3 files changed, 117 insertions(+), 83 deletions(-) diff --git a/music_assistant/server/controllers/media/albums.py b/music_assistant/server/controllers/media/albums.py index e03b3b25..3fcfb325 100644 --- a/music_assistant/server/controllers/media/albums.py +++ b/music_assistant/server/controllers/media/albums.py @@ -107,6 +107,8 @@ class AlbumsController(MediaControllerBase[Album]): await self._match(db_item) # preload album tracks listing (do not load them in the db) for prov_mapping in db_item.provider_mappings: + if not prov_mapping.available: + continue await self._get_provider_album_tracks( prov_mapping.item_id, prov_mapping.provider_instance ) diff --git a/music_assistant/server/providers/tidal/__init__.py b/music_assistant/server/providers/tidal/__init__.py index 53b16439..92791a09 100644 --- a/music_assistant/server/providers/tidal/__init__.py +++ b/music_assistant/server/providers/tidal/__init__.py @@ -7,6 +7,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any +from asyncio_throttle import Throttler from tidalapi import Album as TidalAlbum from tidalapi import Artist as TidalArtist from tidalapi import Config as TidalConfig @@ -163,21 +164,6 @@ async def get_config_entries( ) -async def iter_items(func: Awaitable | Callable, *args, **kwargs) -> AsyncGenerator[Any, None]: - """Yield all items from a larger listing.""" - offset = 0 - while True: - if asyncio.iscoroutinefunction(func): - chunk = await func(*args, **kwargs, offset=offset) - else: - chunk = await asyncio.to_thread(func, *args, **kwargs, offset=offset) - offset += len(chunk) - for item in chunk: - yield item - if len(chunk) < DEFAULT_LIMIT: - break - - class TidalProvider(MusicProvider): """Implementation of a Tidal MusicProvider.""" @@ -188,6 +174,7 @@ class TidalProvider(MusicProvider): """Handle async initialization of the provider.""" self._tidal_user_id = self.config.get_value(CONF_USER_ID) self._tidal_session = await self._get_tidal_session() + self._throttler = Throttler(rate_limit=1, period=0.1) @property def supported_features(self) -> tuple[ProviderFeature, ...]: @@ -241,7 +228,7 @@ class TidalProvider(MusicProvider): """Retrieve all library artists from Tidal.""" tidal_session = await self._get_tidal_session() artist: TidalArtist # satisfy the type checker - async for artist in iter_items( + async for artist in self._iter_items( get_library_artists, tidal_session, self._tidal_user_id, limit=DEFAULT_LIMIT ): yield await self._parse_artist(artist_obj=artist) @@ -250,7 +237,7 @@ class TidalProvider(MusicProvider): """Retrieve all library albums from Tidal.""" tidal_session = await self._get_tidal_session() album: TidalAlbum # satisfy the type checker - async for album in iter_items( + async for album in self._iter_items( get_library_albums, tidal_session, self._tidal_user_id, limit=DEFAULT_LIMIT ): yield await self._parse_album(album_obj=album) @@ -259,7 +246,7 @@ class TidalProvider(MusicProvider): """Retrieve library tracks from Tidal.""" tidal_session = await self._get_tidal_session() track: TidalTrack # satisfy the type checker - async for track in iter_items( + async for track in self._iter_items( get_library_tracks, tidal_session, self._tidal_user_id, limit=DEFAULT_LIMIT ): yield await self._parse_track(track_obj=track) @@ -268,39 +255,44 @@ class TidalProvider(MusicProvider): """Retrieve all library playlists from the provider.""" tidal_session = await self._get_tidal_session() playlist: TidalPlaylist # satisfy the type checker - async for playlist in iter_items(get_library_playlists, tidal_session, self._tidal_user_id): + async for playlist in self._iter_items( + get_library_playlists, tidal_session, self._tidal_user_id + ): yield await self._parse_playlist(playlist_obj=playlist) async def get_album_tracks(self, prov_album_id: str) -> list[Track]: """Get album tracks for given album id.""" tidal_session = await self._get_tidal_session() - return [ - await self._parse_track(track_obj=track) - for track in await get_album_tracks(tidal_session, prov_album_id) - ] + async with self._throttler: + return [ + await self._parse_track(track_obj=track) + for track in await get_album_tracks(tidal_session, prov_album_id) + ] async def get_artist_albums(self, prov_artist_id: str) -> list[Album]: """Get a list of all albums for the given artist.""" tidal_session = await self._get_tidal_session() - return [ - await self._parse_album(album_obj=album) - for album in await get_artist_albums(tidal_session, prov_artist_id) - ] + async with self._throttler: + return [ + await self._parse_album(album_obj=album) + for album in await get_artist_albums(tidal_session, prov_artist_id) + ] async def get_artist_toptracks(self, prov_artist_id: str) -> list[Track]: """Get a list of 10 most popular tracks for the given artist.""" tidal_session = await self._get_tidal_session() - return [ - await self._parse_track(track_obj=track) - for track in await get_artist_toptracks(tidal_session, prov_artist_id) - ] + async with self._throttler: + return [ + await self._parse_track(track_obj=track) + for track in await get_artist_toptracks(tidal_session, prov_artist_id) + ] async def get_playlist_tracks(self, prov_playlist_id: str) -> AsyncGenerator[Track, None]: """Get all playlist tracks for given playlist id.""" tidal_session = await self._get_tidal_session() total_playlist_tracks = 0 track: TidalTrack # satisfy the type checker - async for track_obj in iter_items( + async for track_obj in self._iter_items( get_playlist_tracks, tidal_session, prov_playlist_id, limit=DEFAULT_LIMIT ): total_playlist_tracks += 1 @@ -311,10 +303,11 @@ class TidalProvider(MusicProvider): async def get_similar_tracks(self, prov_track_id: str, limit=25) -> list[Track]: """Get similar tracks for given track id.""" tidal_session = await self._get_tidal_session() - return [ - await self._parse_track(track_obj=track) - for track in await get_similar_tracks(tidal_session, prov_track_id, limit) - ] + async with self._throttler: + return [ + await self._parse_track(track_obj=track) + for track in await get_similar_tracks(tidal_session, prov_track_id, limit) + ] async def library_add(self, prov_item_id: str, media_type: MediaType): """Add item to library.""" @@ -381,46 +374,35 @@ class TidalProvider(MusicProvider): async def get_artist(self, prov_artist_id: str) -> Artist: """Get artist details for given artist id.""" tidal_session = await self._get_tidal_session() - try: - artist = await self._parse_artist( - artist_obj=await get_artist(tidal_session, prov_artist_id), full_details=True + async with self._throttler: + return await self._parse_artist( + artist_obj=await get_artist(tidal_session, prov_artist_id), + full_details=True, ) - except MediaNotFoundError as err: - raise MediaNotFoundError(f"Artist {prov_artist_id} not found") from err - return artist async def get_album(self, prov_album_id: str) -> Album: """Get album details for given album id.""" tidal_session = await self._get_tidal_session() - try: - album = await self._parse_album( + async with self._throttler: + return await self._parse_album( album_obj=await get_album(tidal_session, prov_album_id), full_details=True ) - except MediaNotFoundError as err: - raise MediaNotFoundError(f"Album {prov_album_id} not found") from err - return album async def get_track(self, prov_track_id: str) -> Track: """Get track details for given track id.""" tidal_session = await self._get_tidal_session() - try: - track = await self._parse_track( + async with self._throttler: + return await self._parse_track( track_obj=await get_track(tidal_session, prov_track_id), full_details=True ) - except MediaNotFoundError as err: - raise MediaNotFoundError(f"Track {prov_track_id} not found") from err - return track async def get_playlist(self, prov_playlist_id: str) -> Playlist: """Get playlist details for given playlist id.""" tidal_session = await self._get_tidal_session() - try: - playlist = await self._parse_playlist( + async with self._throttler: + return await self._parse_playlist( playlist_obj=await get_playlist(tidal_session, prov_playlist_id), full_details=True ) - except MediaNotFoundError as err: - raise MediaNotFoundError(f"Playlist {prov_playlist_id} not found") from err - return playlist def get_item_mapping(self, media_type: MediaType, key: str, name: str) -> ItemMapping: """Create a generic item mapping.""" @@ -526,6 +508,7 @@ class TidalProvider(MusicProvider): album.upc = album_obj.universal_product_number album.year = int(album_obj.year) + available = album_obj.available album.add_provider_mapping( ProviderMapping( item_id=album_id, @@ -533,6 +516,7 @@ class TidalProvider(MusicProvider): provider_instance=self.instance_id, content_type=ContentType.FLAC, url=f"http://www.tidal.com/album/{album_id}", + available=available, ) ) # metadata @@ -652,3 +636,20 @@ class TidalProvider(MusicProvider): return item.lyrics return await asyncio.to_thread(inner) + + async def _iter_items( + self, func: Awaitable | Callable, *args, **kwargs + ) -> AsyncGenerator[Any, None]: + """Yield all items from a larger listing.""" + offset = 0 + async with self._throttler: + while True: + if asyncio.iscoroutinefunction(func): + chunk = await func(*args, **kwargs, offset=offset) + else: + chunk = await asyncio.to_thread(func, *args, **kwargs, offset=offset) + offset += len(chunk) + for item in chunk: + yield item + if len(chunk) < DEFAULT_LIMIT: + break diff --git a/music_assistant/server/providers/tidal/helpers.py b/music_assistant/server/providers/tidal/helpers.py index 466fa8e8..7d699353 100644 --- a/music_assistant/server/providers/tidal/helpers.py +++ b/music_assistant/server/providers/tidal/helpers.py @@ -10,6 +10,7 @@ tidalapi: https://github.com/tamland/python-tidal """ import asyncio +import logging from requests import HTTPError from tidalapi import Album as TidalAlbum @@ -23,8 +24,10 @@ from tidalapi import UserPlaylist as TidalUserPlaylist from music_assistant.common.models.enums import MediaType from music_assistant.common.models.errors import MediaNotFoundError +from music_assistant.constants import ROOT_LOGGER_NAME DEFAULT_LIMIT = 50 +LOGGER = logging.getLogger(f"{ROOT_LOGGER_NAME}.tidal.helpers") async def get_library_artists( @@ -72,10 +75,11 @@ async def get_artist(session: TidalSession, prov_artist_id: str) -> TidalArtist: def inner() -> TidalArtist: try: - artist_obj = TidalArtist(session, prov_artist_id) + return TidalArtist(session, prov_artist_id) except HTTPError as err: - raise MediaNotFoundError(f"Artist {prov_artist_id} not found") from err - return artist_obj + if err.response.status_code == 404: + raise MediaNotFoundError(f"Artist {prov_artist_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -84,16 +88,20 @@ async def get_artist_albums(session: TidalSession, prov_artist_id: str) -> list[ """Async wrapper around 3 tidalapi album functions.""" def inner() -> list[TidalAlbum]: - all_albums = [] - albums = TidalArtist(session, prov_artist_id).get_albums(limit=DEFAULT_LIMIT) - eps_singles = TidalArtist(session, prov_artist_id).get_albums_ep_singles( - limit=DEFAULT_LIMIT - ) - compilations = TidalArtist(session, prov_artist_id).get_albums_other(limit=DEFAULT_LIMIT) - all_albums.extend(albums) - all_albums.extend(eps_singles) - all_albums.extend(compilations) - return all_albums + try: + artist_obj = TidalArtist(session, prov_artist_id) + all_albums = [] + albums = artist_obj.get_albums(limit=DEFAULT_LIMIT) + eps_singles = artist_obj.get_albums_ep_singles(limit=DEFAULT_LIMIT) + compilations = artist_obj.get_albums_other(limit=DEFAULT_LIMIT) + all_albums.extend(albums) + all_albums.extend(eps_singles) + all_albums.extend(compilations) + return all_albums + except HTTPError as err: + if err.response.status_code == 404: + raise MediaNotFoundError(f"Artist {prov_artist_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -125,10 +133,11 @@ async def get_album(session: TidalSession, prov_album_id: str) -> TidalAlbum: def inner() -> TidalAlbum: try: - album_obj = TidalAlbum(session, prov_album_id) + return TidalAlbum(session, prov_album_id) except HTTPError as err: - raise MediaNotFoundError(f"Album {prov_album_id} not found") from err - return album_obj + if err.response.status_code == 404: + raise MediaNotFoundError(f"Album {prov_album_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -138,10 +147,11 @@ async def get_track(session: TidalSession, prov_track_id: str) -> TidalTrack: def inner() -> TidalTrack: try: - track_obj = TidalTrack(session, prov_track_id) + return TidalTrack(session, prov_track_id) except HTTPError as err: - raise MediaNotFoundError(f"Track {prov_track_id} not found") from err - return track_obj + if err.response.status_code == 404: + raise MediaNotFoundError(f"Track {prov_track_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -150,7 +160,12 @@ async def get_track_url(session: TidalSession, prov_track_id: str) -> dict[str, """Async wrapper around the tidalapi Track.get_url function.""" def inner() -> dict[str, str]: - return TidalTrack(session, prov_track_id).get_url() + try: + return TidalTrack(session, prov_track_id).get_url() + except HTTPError as err: + if err.response.status_code == 404: + raise MediaNotFoundError(f"Track {prov_track_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -159,7 +174,12 @@ async def get_album_tracks(session: TidalSession, prov_album_id: str) -> list[Ti """Async wrapper around the tidalapi Album.tracks function.""" def inner() -> list[TidalTrack]: - return TidalAlbum(session, prov_album_id).tracks(limit=DEFAULT_LIMIT) + try: + return TidalAlbum(session, prov_album_id).tracks(limit=DEFAULT_LIMIT) + except HTTPError as err: + if err.response.status_code == 404: + raise MediaNotFoundError(f"Album {prov_album_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -191,10 +211,11 @@ async def get_playlist(session: TidalSession, prov_playlist_id: str) -> TidalPla def inner() -> TidalPlaylist: try: - playlist_obj = TidalPlaylist(session, prov_playlist_id) + return TidalPlaylist(session, prov_playlist_id) except HTTPError as err: - raise MediaNotFoundError(f"Playlist {prov_playlist_id} not found") from err - return playlist_obj + if err.response.status_code == 404: + raise MediaNotFoundError(f"Playlist {prov_playlist_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -205,7 +226,12 @@ async def get_playlist_tracks( """Async wrapper around the tidal Playlist.tracks function.""" def inner() -> list[TidalTrack]: - return TidalPlaylist(session, prov_playlist_id).tracks(limit=limit, offset=offset) + try: + return TidalPlaylist(session, prov_playlist_id).tracks(limit=limit, offset=offset) + except HTTPError as err: + if err.response.status_code == 404: + raise MediaNotFoundError(f"Playlist {prov_playlist_id} not found") from err + raise err return await asyncio.to_thread(inner) @@ -239,7 +265,12 @@ async def get_similar_tracks(session: TidalSession, prov_track_id, limit: int) - """Async wrapper around the tidal Track.get_similar_tracks function.""" def inner() -> list[TidalTrack]: - return TidalTrack(session, media_id=prov_track_id).get_track_radio(limit) + try: + return TidalTrack(session, prov_track_id).get_track_radio(limit) + except HTTPError as err: + if err.response.status_code == 404: + raise MediaNotFoundError(f"Track {prov_track_id} not found") from err + raise err return await asyncio.to_thread(inner) -- 2.34.1