From: Marcel van der Veldt Date: Thu, 30 Jun 2022 23:55:59 +0000 (+0200) Subject: Improvements for filesystem provider sync (#391) X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=f6ebdead78e4ce559bcb0c6995b89fead0ab7162;p=music-assistant-server.git Improvements for filesystem provider sync (#391) small fixes for filesystem provider - do not crash on single directory - fix folder.png for albums - overwrite existing metadata on tag changes - keep checksum for cached listings --- diff --git a/music_assistant/controllers/music/albums.py b/music_assistant/controllers/music/albums.py index 98a9d2a2..1843e9a9 100644 --- a/music_assistant/controllers/music/albums.py +++ b/music_assistant/controllers/music/albums.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio import itertools -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from databases import Database as Db @@ -57,14 +57,16 @@ class AlbumsController(MediaControllerBase[Album]): # get results from all providers db_album = await self.get_db_item(item_id) coros = [ - self.get_provider_album_tracks(item.item_id, item.prov_id) + self.get_provider_album_tracks( + item.item_id, item.prov_id, cache_checksum=db_album.metadata.checksum + ) for item in db_album.provider_ids ] tracks = itertools.chain.from_iterable(await asyncio.gather(*coros)) # merge duplicates using a dict final_items: Dict[str, Track] = {} for track in tracks: - key = f".{track.name.lower()}.{track.version}.{track.disc_number}.{track.track_number}" + key = f".{track.name.lower()}.{track.disc_number}.{track.track_number}" if key in final_items: final_items[key].provider_ids.update(track.provider_ids) else: @@ -106,20 +108,23 @@ class AlbumsController(MediaControllerBase[Album]): item_id: str, provider: Optional[ProviderType] = None, provider_id: Optional[str] = None, + cache_checksum: Any = None, ) -> List[Track]: """Return album tracks for the given provider album id.""" prov = self.mass.music.get_provider(provider_id or provider) if not prov: return [] - # prefer cache items (if any) + # prefer cache items (if any) - do not use cache for filesystem cache_key = f"{prov.type.value}.album_tracks.{item_id}" - if cache := await self.mass.cache.get(cache_key): + if cache := await self.mass.cache.get(cache_key, checksum=cache_checksum): return [Track.from_dict(x) for x in cache] # no items in cache - get listing from provider items = await prov.get_album_tracks(item_id) # store (serializable items) in cache self.mass.create_task( - self.mass.cache.set(cache_key, [x.to_dict() for x in items]) + self.mass.cache.set( + cache_key, [x.to_dict() for x in items], checksum=cache_checksum + ) ) return items diff --git a/music_assistant/controllers/music/artists.py b/music_assistant/controllers/music/artists.py index a06f69f9..3ff7d47d 100644 --- a/music_assistant/controllers/music/artists.py +++ b/music_assistant/controllers/music/artists.py @@ -2,7 +2,7 @@ import asyncio import itertools -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from databases import Database as Db @@ -39,7 +39,9 @@ class ArtistsController(MediaControllerBase[Artist]): artist = await self.get(item_id, provider, provider_id) # get results from all providers coros = [ - self.get_provider_artist_toptracks(item.item_id, item.prov_id) + self.get_provider_artist_toptracks( + item.item_id, item.prov_id, cache_checksum=artist.metadata.checksum + ) for item in artist.provider_ids ] tracks = itertools.chain.from_iterable(await asyncio.gather(*coros)) @@ -114,40 +116,52 @@ class ArtistsController(MediaControllerBase[Artist]): ) async def get_provider_artist_toptracks( - self, item_id: str, provider_id: str + self, + item_id: str, + provider: Optional[ProviderType] = None, + provider_id: Optional[str] = None, + cache_checksum: Any = None, ) -> List[Track]: """Return top tracks for an artist on given provider.""" - prov = self.mass.music.get_provider(provider_id) + prov = self.mass.music.get_provider(provider_id or provider) if not prov: return [] # prefer cache items (if any) cache_key = f"{prov.type.value}.artist_toptracks.{item_id}" - if cache := await self.mass.cache.get(cache_key): + if cache := await self.mass.cache.get(cache_key, checksum=cache_checksum): return [Track.from_dict(x) for x in cache] # no items in cache - get listing from provider items = await prov.get_artist_toptracks(item_id) # store (serializable items) in cache self.mass.create_task( - self.mass.cache.set(cache_key, [x.to_dict() for x in items]) + self.mass.cache.set( + cache_key, [x.to_dict() for x in items], checksum=cache_checksum + ) ) return items async def get_provider_artist_albums( - self, item_id: str, provider_id: str + self, + item_id: str, + provider: Optional[ProviderType] = None, + provider_id: Optional[str] = None, + cache_checksum: Any = None, ) -> List[Album]: """Return albums for an artist on given provider.""" - prov = self.mass.music.get_provider(provider_id) + prov = self.mass.music.get_provider(provider_id or provider) if not prov: return [] # prefer cache items (if any) cache_key = f"{prov.type.value}.artist_albums.{item_id}" - if cache := await self.mass.cache.get(cache_key): + if cache := await self.mass.cache.get(cache_key, checksum=cache_checksum): return [Album.from_dict(x) for x in cache] # no items in cache - get listing from provider items = await prov.get_artist_albums(item_id) # store (serializable items) in cache self.mass.create_task( - self.mass.cache.set(cache_key, [x.to_dict() for x in items]) + self.mass.cache.set( + cache_key, [x.to_dict() for x in items], checksum=cache_checksum + ) ) return items diff --git a/music_assistant/controllers/music/tracks.py b/music_assistant/controllers/music/tracks.py index 01fa38f2..41631ab9 100644 --- a/music_assistant/controllers/music/tracks.py +++ b/music_assistant/controllers/music/tracks.py @@ -149,7 +149,9 @@ class TracksController(MediaControllerBase[Track]): # no existing match found: insert new item track_artists = await self._get_track_artists(item, db=db) - track_albums = await self._get_track_albums(item, db=db) + track_albums = await self._get_track_albums( + item, overwrite=overwrite_existing, db=db + ) new_item = await self.mass.database.insert( self.db_table, { @@ -187,7 +189,7 @@ class TracksController(MediaControllerBase[Track]): metadata.last_refresh = None # we store a mapping to artists/albums on the item for easier access/listings track_artists = await self._get_track_artists(item, db=db) - track_albums = await self._get_track_albums(item, db=db) + track_albums = await self._get_track_albums(item, overwrite=True, db=db) else: metadata = cur_item.metadata.update(item.metadata, overwrite) provider_ids = {*cur_item.provider_ids, *item.provider_ids} @@ -237,6 +239,7 @@ class TracksController(MediaControllerBase[Track]): self, base_track: Track, upd_track: Optional[Track] = None, + overwrite: bool = False, db: Optional[Db] = None, ) -> List[TrackAlbumMapping]: """Extract all (unique) albums of track as TrackAlbumMapping.""" @@ -248,7 +251,9 @@ class TracksController(MediaControllerBase[Track]): track_albums = base_track.albums # append update item album if needed if upd_track and upd_track.album: - mapping = await self._get_album_mapping(upd_track.album, db=db) + mapping = await self._get_album_mapping( + upd_track.album, overwrite=overwrite, db=db + ) mapping = TrackAlbumMapping.from_dict( { **mapping.to_dict(), @@ -260,7 +265,9 @@ class TracksController(MediaControllerBase[Track]): track_albums.append(mapping) # append base item album if needed elif base_track and base_track.album: - mapping = await self._get_album_mapping(base_track.album, db=db) + mapping = await self._get_album_mapping( + base_track.album, overwrite=overwrite, db=db + ) mapping = TrackAlbumMapping.from_dict( { **mapping.to_dict(), @@ -274,7 +281,10 @@ class TracksController(MediaControllerBase[Track]): return track_albums async def _get_album_mapping( - self, album: Union[Album, ItemMapping], db: Optional[Db] = None + self, + album: Union[Album, ItemMapping], + overwrite: bool = False, + db: Optional[Db] = None, ) -> ItemMapping: """Extract (database) album as ItemMapping.""" if album.provider == ProviderType.DATABASE: @@ -287,7 +297,9 @@ class TracksController(MediaControllerBase[Track]): ): return ItemMapping.from_item(db_album) - db_album = await self.mass.music.albums.add_db_item(album, db=db) + db_album = await self.mass.music.albums.add_db_item( + album, overwrite_existing=overwrite, db=db + ) return ItemMapping.from_item(db_album) async def _get_artist_mapping( diff --git a/music_assistant/helpers/compare.py b/music_assistant/helpers/compare.py index ef4a0292..abe4028d 100644 --- a/music_assistant/helpers/compare.py +++ b/music_assistant/helpers/compare.py @@ -176,13 +176,23 @@ def compare_track(left_track: Track, right_track: Track): # album is required for track linking if left_track.album is None or right_track.album is None: return False - # track name and version must match + # track name must match if not left_track.sort_name: left_track.sort_name = create_clean_string(left_track.name) if not right_track.sort_name: right_track.sort_name = create_clean_string(right_track.name) if left_track.sort_name != right_track.sort_name: return False + # exact albumtrack match = 100% match + if ( + compare_album(left_track.album, right_track.album) + and left_track.track_number + and right_track.track_number + and left_track.disc_number == right_track.disc_number + and left_track.track_number == right_track.track_number + ): + return True + # track version must match if not compare_version(left_track.version, right_track.version): return False # track artist(s) must match @@ -192,8 +202,6 @@ def compare_track(left_track: Track, right_track: Track): if not compare_explicit(left_track.metadata, right_track.metadata): return False # exact album match = 100% match - if compare_album(left_track.album, right_track.album): - return True if left_track.albums and right_track.albums: for left_album in left_track.albums: for right_album in right_track.albums: @@ -201,7 +209,7 @@ def compare_track(left_track: Track, right_track: Track): return True # fallback: both albums are compilations and (near-exact) track duration match if ( - abs(left_track.duration - right_track.duration) <= 1 + abs(left_track.duration - right_track.duration) <= 2 and left_track.album.album_type in (AlbumType.UNKNOWN, AlbumType.COMPILATION) and right_track.album.album_type in (AlbumType.UNKNOWN, AlbumType.COMPILATION) ): diff --git a/music_assistant/models/media_controller.py b/music_assistant/models/media_controller.py index e7b81685..dbc48d99 100644 --- a/music_assistant/models/media_controller.py +++ b/music_assistant/models/media_controller.py @@ -164,8 +164,9 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): cache_key = ( f"{prov.type.value}.search.{self.media_type.value}.{search_query}.{limit}" ) - if cache := await self.mass.cache.get(cache_key): - return [media_from_dict(x) for x in cache] + if not prov.type.is_file(): # do not cache filesystem results + if cache := await self.mass.cache.get(cache_key): + return [media_from_dict(x) for x in cache] # no items in cache - get listing from provider items = await prov.search( search_query, diff --git a/music_assistant/music_providers/filesystem.py b/music_assistant/music_providers/filesystem.py index e0c3097f..4c297e5c 100644 --- a/music_assistant/music_providers/filesystem.py +++ b/music_assistant/music_providers/filesystem.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio +import logging import os import urllib.parse from contextlib import asynccontextmanager @@ -54,6 +55,7 @@ CONTENT_TYPE_EXT = { "aiff": ContentType.AIFF, } SCHEMA_VERSION = 17 +LOGGER = logging.getLogger(__name__) async def scantree(path: str) -> AsyncGenerator[os.DirEntry, None]: @@ -63,12 +65,17 @@ async def scantree(path: str) -> AsyncGenerator[os.DirEntry, None]: return entry.is_dir(follow_symlinks=False) loop = asyncio.get_running_loop() - for entry in await loop.run_in_executor(None, os.scandir, path): - if await loop.run_in_executor(None, is_dir, entry): - async for subitem in scantree(entry.path): - yield subitem - else: - yield entry + try: + entries = await loop.run_in_executor(None, os.scandir, path) + except (OSError, PermissionError) as err: + LOGGER.warning("Skip folder %s: %s", path, str(err)) + else: + for entry in entries: + if await loop.run_in_executor(None, is_dir, entry): + async for subitem in scantree(entry.path): + yield subitem + else: + yield entry def split_items(org_str: str) -> Tuple[str]: @@ -165,8 +172,12 @@ class FileSystemProvider(MusicProvider): continue if track := await self._parse_track(entry.path): + # set checksum on track to invalidate any cached listings + track.metadata.checksum = checksum # process album if track.album: + # set checksum on album to invalidate cached albumtracks listings etc + track.album.metadata.checksum = checksum db_album = await self.mass.music.albums.add_db_item( track.album, overwrite_existing=True, db=db ) @@ -176,6 +187,8 @@ class FileSystemProvider(MusicProvider): ) # process (album)artist if track.album.artist: + # set checksum on albumartist to invalidate cached artisttracks listings etc + track.album.artist.metadata.checksum = checksum db_artist = await self.mass.music.artists.add_db_item( track.album.artist, db=db ) @@ -780,7 +793,7 @@ class FileSystemProvider(MusicProvider): for img_type in ImageType: if img_type.value in _filepath: images.append(MediaItemImage(img_type, _filepath, True)) - elif _filename == "folder.jpg": + elif "folder." in _filepath: images.append(MediaItemImage(ImageType.THUMB, _filepath, True)) if images: album.metadata.images = images