optimize caching
authorMarcel van der Veldt <m.vanderveldt@outlook.com>
Mon, 15 Apr 2024 21:37:04 +0000 (23:37 +0200)
committerMarcel van der Veldt <m.vanderveldt@outlook.com>
Mon, 15 Apr 2024 21:37:04 +0000 (23:37 +0200)
music_assistant/common/models/media_items.py
music_assistant/server/controllers/media/albums.py
music_assistant/server/controllers/media/artists.py
music_assistant/server/controllers/media/base.py
music_assistant/server/controllers/media/playlists.py
music_assistant/server/models/music_provider.py
music_assistant/server/providers/builtin/__init__.py

index 3ebffe88c789387d601a92cdf84b0c005650df12..3dfbdce3af6bc3bba034382d756ab5ac6b3e8e97 100644 (file)
@@ -147,13 +147,15 @@ class MediaItemImage(DataClassDictMixin):
         """Check equality of two items."""
         return self.__hash__() == other.__hash__()
 
-    def __post_init__(self):
-        """Call after init."""
+    @classmethod
+    def __pre_deserialize__(cls, d: dict[Any, Any]) -> dict[Any, Any]:
+        """Handle actions before deserialization."""
         # migrate from url provider --> builtin
         # TODO: remove this after 2.0 is launched
-        if self.provider == "url":
-            self.provider = "builtin"
-            self.remotely_accessible = True
+        if d["provider"] == "url":
+            d["provider"] = "builtin"
+            d["remotely_accessible"] = True
+        return d
 
 
 @dataclass(frozen=True, kw_only=True)
index 4674ae1f5fa89b9d4cfaca9df4b0c0cf097a8814..aa1e5a8f50386e88193951859dbf4a597f73f9dd 100644 (file)
@@ -279,7 +279,7 @@ class AlbumsController(MediaControllerBase[Album]):
 
         full_album = await self.get_provider_item(item_id, provider_instance_id_or_domain)
         # prefer cache items (if any) for streaming providers only
-        cache_key = f"{prov.instance_id}.albumtracks.{item_id}"
+        cache_key = f"{prov.lookup_key}.albumtracks.{item_id}"
         if prov.is_streaming_provider and (cache := await self.mass.cache.get(cache_key)):
             return [AlbumTrack.from_dict(x) for x in cache]
         # no items in cache - get listing from provider
index 13c6b3291ec81fca3271453149940de2340fe49b..c3c69dcfd9893d9a3055a98aed5056fa15900557 100644 (file)
@@ -242,8 +242,10 @@ class ArtistsController(MediaControllerBase[Artist]):
             if not provider.is_streaming_provider:
                 # matching on unique providers is pointless as they push (all) their content to MA
                 continue
-            if await self._match(db_artist, provider):
-                cur_provider_domains.add(provider.domain)
+            for unpack_item_mapping in (False, True):
+                if await self._match(db_artist, provider, unpack_item_mapping):
+                    cur_provider_domains.add(provider.domain)
+                    break
             else:
                 self.logger.debug(
                     "Could not find match for Artist %s on provider %s",
@@ -263,7 +265,7 @@ class ArtistsController(MediaControllerBase[Artist]):
         if prov is None:
             return []
         # prefer cache items (if any) - for streaming providers
-        cache_key = f"{prov.instance_id}.artist_toptracks.{item_id}"
+        cache_key = f"{prov.lookup_key}.artist_toptracks.{item_id}"
         if prov.is_streaming_provider and (cache := await self.mass.cache.get(cache_key)):
             return [Track.from_dict(x) for x in cache]
         # no items in cache - get listing from provider
@@ -309,7 +311,7 @@ class ArtistsController(MediaControllerBase[Artist]):
         if prov is None:
             return []
         # prefer cache items (if any)
-        cache_key = f"{prov.instance_id}.artist_albums.{item_id}"
+        cache_key = f"{prov.lookup_key}.artist_albums.{item_id}"
         if prov.is_streaming_provider and (cache := await self.mass.cache.get(cache_key)):
             return [Album.from_dict(x) for x in cache]
         # no items in cache - get listing from provider
@@ -418,7 +420,9 @@ class ArtistsController(MediaControllerBase[Artist]):
         msg = "No Music Provider found that supports requesting similar tracks."
         raise UnsupportedFeaturedException(msg)
 
-    async def _match(self, db_artist: Artist, provider: MusicProvider) -> bool:
+    async def _match(
+        self, db_artist: Artist, provider: MusicProvider, unpack_item_mapping: bool = False
+    ) -> bool:
         """Try to find matching artists on given provider for the provided (database) artist."""
         self.logger.debug("Trying to match artist %s on provider %s", db_artist.name, provider.name)
         # try to get a match with some reference tracks of this artist
@@ -432,8 +436,10 @@ class ArtistsController(MediaControllerBase[Artist]):
         for ref_track in ref_tracks:
             # make sure we have a full track
             if isinstance(ref_track.album, ItemMapping):
+                if not unpack_item_mapping:
+                    continue
                 try:
-                    ref_track = await self.mass.music.tracks.get_provider_item(  # noqa: PLW2901
+                    ref_track = await self.mass.music.tracks.get(  # noqa: PLW2901
                         ref_track.item_id, ref_track.provider
                     )
                 except MediaNotFoundError:
index 4b744766de313fecd39d8c1ceb977c650fed4fd0..ff61dbdc0c59128a120a776f50da111073a3f1d3 100644 (file)
@@ -10,7 +10,11 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from music_assistant.common.helpers.json import json_loads, serialize_to_json
 from music_assistant.common.models.enums import EventType, ExternalID, MediaType, ProviderFeature
-from music_assistant.common.models.errors import InvalidDataError, MediaNotFoundError
+from music_assistant.common.models.errors import (
+    InvalidDataError,
+    MediaNotFoundError,
+    ProviderUnavailableError,
+)
 from music_assistant.common.models.media_items import (
     Album,
     Artist,
@@ -236,7 +240,7 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta):
             return []
 
         # prefer cache items (if any)
-        cache_key = f"{prov.instance_id}.search.{self.media_type.value}.{search_query}.{limit}"
+        cache_key = f"{prov.lookup_key}.search.{self.media_type.value}.{search_query}.{limit}"
         if cache := await self.mass.cache.get(cache_key):
             return [media_from_dict(x) for x in cache]
         # no items in cache - get listing from provider
@@ -430,9 +434,9 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta):
         fallback: ItemMapping | ItemCls = None,
     ) -> ItemCls:
         """Return item details for the given provider item id."""
-        cache_key = (
-            f"provider_item.{self.media_type.value}.{provider_instance_id_or_domain}.{item_id}"
-        )
+        if not (provider := self.mass.get_provider(provider_instance_id_or_domain)):
+            raise ProviderUnavailableError(f"{provider_instance_id_or_domain} is not available")
+        cache_key = f"provider_item.{self.media_type.value}.{provider.lookup_key}.{item_id}"
         if provider_instance_id_or_domain == "library":
             return await self.get_library_item(item_id)
         if not force_refresh and (cache := await self.mass.cache.get(cache_key)):
index c00518e0ff0e2ca7948bf04b82eeaba21eee1134..a352aed6807c1c0f26c88df59f1f5b78130fc8a0 100644 (file)
@@ -264,7 +264,7 @@ class PlaylistController(MediaControllerBase[Playlist]):
         # actually add the tracks to the playlist on the provider
         await playlist_prov.add_playlist_tracks(playlist_prov_map.item_id, list(ids_to_add))
         # invalidate cache so tracks get refreshed
-        cache_key = f"{playlist_prov.instance_id}.playlist.{playlist_prov_map.item_id}.tracks"
+        cache_key = f"{playlist_prov.lookup_key}.playlist.{playlist_prov_map.item_id}.tracks"
         await self.mass.cache.delete(cache_key)
 
     async def add_playlist_track(self, db_playlist_id: str | int, track_uri: str) -> None:
@@ -293,7 +293,7 @@ class PlaylistController(MediaControllerBase[Playlist]):
                 continue
             await provider.remove_playlist_tracks(prov_mapping.item_id, positions_to_remove)
         # invalidate cache so tracks get refreshed
-        cache_key = f"{provider.instance_id}.playlist.{prov_mapping.item_id}.tracks"
+        cache_key = f"{provider.lookup_key}.playlist.{prov_mapping.item_id}.tracks"
         await self.mass.cache.delete(cache_key)
 
     async def _add_library_item(self, item: Playlist) -> Playlist:
@@ -334,7 +334,7 @@ class PlaylistController(MediaControllerBase[Playlist]):
         if not provider:
             return
         # prefer cache items (if any)
-        cache_key = f"{provider.instance_id}.playlist.{item_id}.tracks"
+        cache_key = f"{provider.lookup_key}.playlist.{item_id}.tracks"
         if cache := await self.mass.cache.get(cache_key, checksum=cache_checksum):
             for track_dict in cache:
                 yield PlaylistTrack.from_dict(track_dict)
@@ -346,6 +346,11 @@ class PlaylistController(MediaControllerBase[Playlist]):
             assert item.position is not None, "Playlist items require position to be set"
             yield item
             all_items.append(item)
+            # if this is a complete track object, pre-cache it as
+            # that will save us an (expensive) lookup later
+            if item.image and item.artist_str and item.album and provider.domain != "builtin":
+                cache_key = f"provider_item.track.{provider.lookup_key}.{item_id}"
+                await self.mass.cache.set(cache_key, item.to_dict())
         # store (serializable items) in cache
         if cache_checksum != "no_cache":
             self.mass.create_task(
index fd0e140a19965acf702087cbe32b82b87345f0b0..14fb2d9a27daac321b9b09566947349b73d5634b 100644 (file)
@@ -398,7 +398,6 @@ class MusicProvider(Provider):
                         # create full db item
                         # note that we skip the metadata lookup purely to speed up the sync
                         # the additional metadata is then lazy retrieved afterwards
-
                         prov_item.favorite = True
                         extra_kwargs = (
                             {"add_album_tracks": True} if media_type == MediaType.ALBUM else {}
index de899438fbb92966a3b7846337568abb91f0eb55..d0ea2e7f40c7722394e1a57a0949ffd74545d09c 100644 (file)
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, NotRequired, TypedDict
 
 import shortuuid
 
+from music_assistant.common.helpers.uri import parse_uri
 from music_assistant.common.models.config_entries import ConfigEntry
 from music_assistant.common.models.enums import (
     ConfigEntryType,
@@ -17,7 +18,11 @@ from music_assistant.common.models.enums import (
     ProviderFeature,
     StreamType,
 )
-from music_assistant.common.models.errors import InvalidDataError, MediaNotFoundError
+from music_assistant.common.models.errors import (
+    InvalidDataError,
+    MediaNotFoundError,
+    ProviderUnavailableError,
+)
 from music_assistant.common.models.media_items import (
     AlbumTrack,
     Artist,
@@ -349,12 +354,18 @@ class BuiltinProvider(MusicProvider):
         # user created universal playlist
         conf_key = f"{CONF_KEY_PLAYLIST_ITEMS}/{prov_playlist_id}"
         playlist_items: list[str] = self.mass.config.get(conf_key, [])
-        for count, playlist_item_uri in enumerate(playlist_items, 1):
+        for count, uri in enumerate(playlist_items, 1):
             try:
-                base_item = await self.mass.music.get_item_by_uri(playlist_item_uri)
+                # get the provider item and not the full track from a regular 'get' call
+                # as we only need basic track info here
+                media_type, provider_instance_id_or_domain, item_id = await parse_uri(uri)
+                media_controller = self.mass.music.get_controller(media_type)
+                base_item = await media_controller.get_provider_item(
+                    item_id, provider_instance_id_or_domain
+                )
                 yield PlaylistTrack.from_dict({**base_item.to_dict(), "position": count})
-            except (MediaNotFoundError, InvalidDataError) as err:
-                self.logger.warning("Skipping item in playlist: %s:%s", playlist_item_uri, str(err))
+            except (MediaNotFoundError, InvalidDataError, ProviderUnavailableError) as err:
+                self.logger.warning("Skipping item in playlist: %s:%s", uri, str(err))
 
     async def add_playlist_tracks(self, prov_playlist_id: str, prov_track_ids: list[str]) -> None:
         """Add track(s) to playlist."""