From: Marcel van der Veldt Date: Thu, 23 Jun 2022 09:29:19 +0000 (+0200) Subject: Improve cache for items from music providers (#383) X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=8ad804c4bf90798eb86f8d5d8695b7f96f062afd;p=music-assistant-server.git Improve cache for items from music providers (#383) remove cache from provider implementations and have the logic at global/abstract level --- diff --git a/music_assistant/controllers/music/albums.py b/music_assistant/controllers/music/albums.py index b0b78e3c..c8953bbd 100644 --- a/music_assistant/controllers/music/albums.py +++ b/music_assistant/controllers/music/albums.py @@ -107,7 +107,17 @@ class AlbumsController(MediaControllerBase[Album]): prov = self.mass.music.get_provider(provider_id or provider) if not prov: return [] - return await prov.get_album_tracks(item_id) + # prefer cache items (if any) + cache_key = f"{prov.type.value}.album_tracks.{item_id}" + if cache := await self.mass.cache.get(cache_key): + return [Track.from_dict(x) for x in cache] + # no items in cache - get listing from provider + items = await prov.get_album_tracks(item_id) + # store (serializable items) in cache + self.mass.create_task( + self.mass.cache.set(cache_key, [x.to_dict() for x in items]) + ) + return items async def add_db_item( self, item: Album, overwrite_existing: bool = False, db: Optional[Db] = None diff --git a/music_assistant/controllers/music/artists.py b/music_assistant/controllers/music/artists.py index 3be1cf4a..df27a1ec 100644 --- a/music_assistant/controllers/music/artists.py +++ b/music_assistant/controllers/music/artists.py @@ -117,19 +117,39 @@ class ArtistsController(MediaControllerBase[Artist]): self, item_id: str, provider_id: str ) -> List[Track]: """Return top tracks for an artist on given provider.""" - provider = self.mass.music.get_provider(provider_id) - if not provider: + prov = self.mass.music.get_provider(provider_id) + if not prov: return [] - return await provider.get_artist_toptracks(item_id) + # prefer cache items (if any) + cache_key = f"{prov.type.value}.artist_albums.{item_id}" + if cache := await self.mass.cache.get(cache_key): + return [Track.from_dict(x) for x in cache] + # no items in cache - get listing from provider + items = await prov.get_artist_toptracks(item_id) + # store (serializable items) in cache + self.mass.create_task( + self.mass.cache.set(cache_key, [x.to_dict() for x in items]) + ) + return items async def get_provider_artist_albums( self, item_id: str, provider_id: str ) -> List[Album]: """Return albums for an artist on given provider.""" - provider = self.mass.music.get_provider(provider_id) - if not provider: + prov = self.mass.music.get_provider(provider_id) + if not prov: return [] - return await provider.get_artist_albums(item_id) + # prefer cache items (if any) + cache_key = f"{prov.type.value}.artist_albums.{item_id}" + if cache := await self.mass.cache.get(cache_key): + return [Album.from_dict(x) for x in cache] + # no items in cache - get listing from provider + items = await prov.get_artist_albums(item_id) + # store (serializable items) in cache + self.mass.create_task( + self.mass.cache.set(cache_key, [x.to_dict() for x in items]) + ) + return items async def add_db_item( self, item: Artist, overwrite_existing: bool = False, db: Optional[Db] = None diff --git a/music_assistant/controllers/music/playlists.py b/music_assistant/controllers/music/playlists.py index eeca5a4e..6c39b3d4 100644 --- a/music_assistant/controllers/music/playlists.py +++ b/music_assistant/controllers/music/playlists.py @@ -34,17 +34,26 @@ class PlaylistController(MediaControllerBase[Playlist]): provider_id: Optional[str] = None, ) -> List[Track]: """Return playlist tracks for the given provider playlist id.""" - if provider == ProviderType.DATABASE or provider_id == "database": - playlist = await self.get_db_item(item_id) - prov = next(x for x in playlist.provider_ids) - item_id = prov.item_id - provider_id = prov.prov_id + playlist = await self.get(item_id, provider, provider_id) + prov = next(x for x in playlist.provider_ids) + prov_playlist_id = prov.item_id + provider_id = prov.prov_id provider = self.mass.music.get_provider(provider_id or provider) if not provider: return [] - - return await provider.get_playlist_tracks(item_id) + # prefer cache for playlist tracks - use checksum from playlist + cache_key = f"{provider.value}.playlist_tracks.{prov_playlist_id}" + cache_checksum = playlist.metadata.checksum + if cache := await self.mass.cache.get(cache_key, cache_checksum): + return [Track.from_dict(x) for x in cache] + # no items in cache - get listing from provider + items = await provider.get_playlist_tracks(prov_playlist_id) + # store (serializable items) in cache + self.mass.create_task( + self.mass.cache.set(cache_key, [x.to_dict() for x in items], cache_checksum) + ) + return items async def add(self, item: Playlist) -> Playlist: """Add playlist to local db and return the new database item.""" diff --git a/music_assistant/music_providers/filesystem.py b/music_assistant/music_providers/filesystem.py index 16058877..5d561a92 100644 --- a/music_assistant/music_providers/filesystem.py +++ b/music_assistant/music_providers/filesystem.py @@ -325,12 +325,6 @@ class FileSystemProvider(MusicProvider): playlist_path = await self.get_filepath(MediaType.PLAYLIST, prov_playlist_id) if not await self.exists(playlist_path): raise MediaNotFoundError(f"Playlist path does not exist: {playlist_path}") - getmtime = wrap(os.path.getmtime) - mtime = await getmtime(playlist_path) - checksum = f"{SCHEMA_VERSION}.{int(mtime)}" - cache_key = f"playlist_{self.id}_tracks_{prov_playlist_id}" - if cache := await self.mass.cache.get(cache_key, checksum): - return [Track.from_dict(x) for x in cache] playlist_base_path = Path(playlist_path).parent index = 0 try: @@ -349,7 +343,6 @@ class FileSystemProvider(MusicProvider): self.logger.warning( "Error while parsing playlist %s", playlist_path, exc_info=err ) - await self.mass.cache.set(cache_key, [x.to_dict() for x in result], checksum) return result async def _parse_playlist_line(self, line: str, playlist_path: str) -> Track | None: @@ -815,6 +808,10 @@ class FileSystemProvider(MusicProvider): ) ) playlist.owner = self._attr_name + getmtime = wrap(os.path.getmtime) + mtime = await getmtime(playlist_path) + checksum = f"{SCHEMA_VERSION}.{int(mtime)}" + playlist.metadata.checksum = checksum return playlist async def exists(self, file_path: str) -> bool: diff --git a/music_assistant/music_providers/qobuz.py b/music_assistant/music_providers/qobuz.py index c55b50c0..a58404dd 100644 --- a/music_assistant/music_providers/qobuz.py +++ b/music_assistant/music_providers/qobuz.py @@ -14,7 +14,6 @@ from music_assistant.helpers.app_vars import ( # pylint: disable=no-name-in-mod app_var, ) from music_assistant.helpers.audio import get_http_stream -from music_assistant.helpers.cache import use_cache from music_assistant.helpers.util import parse_title_and_version, try_parse_int from music_assistant.models.enums import ProviderType from music_assistant.models.errors import LoginFailed, MediaNotFoundError @@ -114,36 +113,28 @@ class QobuzProvider(MusicProvider): async def get_library_artists(self) -> AsyncGenerator[Artist, None]: """Retrieve all library artists from Qobuz.""" endpoint = "favorite/getUserFavorites" - for item in await self._get_all_items( - endpoint, key="artists", type="artists", skip_cache=True - ): + for item in await self._get_all_items(endpoint, key="artists", type="artists"): if item and item["id"]: yield await self._parse_artist(item) async def get_library_albums(self) -> AsyncGenerator[Album, None]: """Retrieve all library albums from Qobuz.""" endpoint = "favorite/getUserFavorites" - for item in await self._get_all_items( - endpoint, key="albums", type="albums", skip_cache=True - ): + for item in await self._get_all_items(endpoint, key="albums", type="albums"): if item and item["id"]: yield await self._parse_album(item) async def get_library_tracks(self) -> AsyncGenerator[Track, None]: """Retrieve library tracks from Qobuz.""" endpoint = "favorite/getUserFavorites" - for item in await self._get_all_items( - endpoint, key="tracks", type="tracks", skip_cache=True - ): + for item in await self._get_all_items(endpoint, key="tracks", type="tracks"): if item and item["id"]: yield await self._parse_track(item) async def get_library_playlists(self) -> AsyncGenerator[Playlist, None]: """Retrieve all library playlists from the provider.""" endpoint = "playlist/getUserPlaylists" - for item in await self._get_all_items( - endpoint, key="playlists", skip_cache=True - ): + for item in await self._get_all_items(endpoint, key="playlists"): if item and item["id"]: yield await self._parse_playlist(item) @@ -198,7 +189,6 @@ class QobuzProvider(MusicProvider): async def get_playlist_tracks(self, prov_playlist_id) -> List[Track]: """Get all playlist tracks for given playlist id.""" - playlist = await self.get_playlist(prov_playlist_id) endpoint = "playlist/get" return [ await self._parse_track(item) @@ -207,7 +197,6 @@ class QobuzProvider(MusicProvider): key="tracks", playlist_id=prov_playlist_id, extra="tracks", - cache_checksum=playlist.metadata.checksum, ) if (item and item["id"]) ] @@ -333,7 +322,6 @@ class QobuzProvider(MusicProvider): format_id=format_id, track_id=item_id, intent="stream", - skip_cache=True, ) if result and result.get("url"): streamdata = result @@ -638,7 +626,6 @@ class QobuzProvider(MusicProvider): self.mass.metadata.preferred_language = details["user"]["country_code"] return details["user_auth_token"] - @use_cache(3600 * 24) async def _get_all_items(self, endpoint, key="tracks", **kwargs): """Get all items from a paged list.""" limit = 50 @@ -647,7 +634,7 @@ class QobuzProvider(MusicProvider): while True: kwargs["limit"] = limit kwargs["offset"] = offset - result = await self._get_data(endpoint, skip_cache=True, **kwargs) + result = await self._get_data(endpoint, **kwargs) offset += limit if not result: break @@ -660,7 +647,6 @@ class QobuzProvider(MusicProvider): break return all_items - @use_cache(3600 * 2) async def _get_data(self, endpoint, sign_request=False, **kwargs): """Get data from api.""" url = f"http://www.qobuz.com/api.json/0.2/{endpoint}" diff --git a/music_assistant/music_providers/spotify.py b/music_assistant/music_providers/spotify.py index f9da7013..891ad459 100644 --- a/music_assistant/music_providers/spotify.py +++ b/music_assistant/music_providers/spotify.py @@ -16,7 +16,6 @@ from asyncio_throttle import Throttler from music_assistant.helpers.app_vars import ( # noqa # pylint: disable=no-name-in-module app_var, ) -from music_assistant.helpers.cache import use_cache from music_assistant.helpers.process import AsyncProcess from music_assistant.helpers.util import parse_title_and_version from music_assistant.models.enums import ProviderType @@ -137,7 +136,9 @@ class SpotifyProvider(MusicProvider): endpoint = "me/following" while True: spotify_artists = await self._get_data( - endpoint, type="artist", limit=50, skip_cache=True + endpoint, + type="artist", + limit=50, ) for item in spotify_artists["artists"]["items"]: if item and item["id"]: @@ -150,19 +151,19 @@ class SpotifyProvider(MusicProvider): async def get_library_albums(self) -> AsyncGenerator[Album, None]: """Retrieve library albums from the provider.""" - for item in await self._get_all_items("me/albums", skip_cache=True): + for item in await self._get_all_items("me/albums"): if item["album"] and item["album"]["id"]: yield await self._parse_album(item["album"]) async def get_library_tracks(self) -> AsyncGenerator[Track, None]: """Retrieve library tracks from the provider.""" - for item in await self._get_all_items("me/tracks", skip_cache=True): + for item in await self._get_all_items("me/tracks"): if item and item["track"]["id"]: yield await self._parse_track(item["track"]) async def get_library_playlists(self) -> AsyncGenerator[Playlist, None]: """Retrieve playlists from the provider.""" - for item in await self._get_all_items("me/playlists", skip_cache=True): + for item in await self._get_all_items("me/playlists"): if item and item["id"]: yield await self._parse_playlist(item) @@ -196,12 +197,10 @@ class SpotifyProvider(MusicProvider): async def get_playlist_tracks(self, prov_playlist_id) -> List[Track]: """Get all playlist tracks for given playlist id.""" - playlist = await self.get_playlist(prov_playlist_id) return [ await self._parse_track(item["track"]) for item in await self._get_all_items( f"playlists/{prov_playlist_id}/tracks", - cache_checksum=playlist.metadata.checksum, ) if (item and item["track"] and item["track"]["id"]) ] @@ -580,7 +579,6 @@ class SpotifyProvider(MusicProvider): return tokeninfo return None - @use_cache(3600 * 24) async def _get_all_items(self, endpoint, key="items", **kwargs) -> List[dict]: """Get all items from a paged list.""" limit = 50 @@ -589,7 +587,7 @@ class SpotifyProvider(MusicProvider): while True: kwargs["limit"] = limit kwargs["offset"] = offset - result = await self._get_data(endpoint, skip_cache=True, **kwargs) + result = await self._get_data(endpoint, **kwargs) offset += limit if not result or key not in result or not result[key]: break @@ -600,7 +598,6 @@ class SpotifyProvider(MusicProvider): break return all_items - @use_cache(3600 * 2) async def _get_data(self, endpoint, **kwargs): """Get data from api.""" url = f"https://api.spotify.com/v1/{endpoint}" diff --git a/music_assistant/music_providers/tunein.py b/music_assistant/music_providers/tunein.py index e39c2cc7..1a7eab7d 100644 --- a/music_assistant/music_providers/tunein.py +++ b/music_assistant/music_providers/tunein.py @@ -6,7 +6,6 @@ from typing import AsyncGenerator, List, Optional from asyncio_throttle import Throttler from music_assistant.helpers.audio import get_radio_stream -from music_assistant.helpers.cache import use_cache from music_assistant.helpers.util import create_clean_string from music_assistant.models.enums import ProviderType from music_assistant.models.errors import LoginFailed, MediaNotFoundError @@ -176,7 +175,6 @@ class TuneInProvider(MusicProvider): ): yield chunk - @use_cache(3600 * 2) async def __get_data(self, endpoint: str, **kwargs): """Get data from api.""" if endpoint.startswith("http"):