From 3174b3bd6d729d813e28e54d54ac8721eab24f32 Mon Sep 17 00:00:00 2001 From: Marcel van der Veldt Date: Tue, 9 Jul 2024 20:38:07 +0200 Subject: [PATCH] tweak metadata retrieval --- .../server/controllers/media/albums.py | 2 +- .../server/controllers/media/base.py | 25 ++------ .../server/controllers/media/tracks.py | 3 +- .../server/controllers/metadata.py | 64 +++++++++++-------- music_assistant/server/controllers/music.py | 12 +++- .../server/models/music_provider.py | 4 +- .../providers/filesystem_local/__init__.py | 56 ---------------- .../server/providers/filesystem_local/base.py | 3 +- .../providers/filesystem_smb/__init__.py | 1 - 9 files changed, 56 insertions(+), 114 deletions(-) diff --git a/music_assistant/server/controllers/media/albums.py b/music_assistant/server/controllers/media/albums.py index c6f26069..e5b45ba9 100644 --- a/music_assistant/server/controllers/media/albums.py +++ b/music_assistant/server/controllers/media/albums.py @@ -367,7 +367,7 @@ class AlbumsController(MediaControllerBase[Album]): if not db_artist or overwrite: db_artist = await self.mass.music.artists.add_item_to_library( - artist, metadata_lookup=False, overwrite_existing=overwrite + artist, overwrite_existing=overwrite ) # write (or update) record in album_artists table await self.mass.music.database.insert_or_replace( diff --git a/music_assistant/server/controllers/media/base.py b/music_assistant/server/controllers/media/base.py index f94ec46b..9e3444ea 100644 --- a/music_assistant/server/controllers/media/base.py +++ b/music_assistant/server/controllers/media/base.py @@ -91,23 +91,20 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): async def add_item_to_library( self, item: ItemCls, - metadata_lookup: bool = True, overwrite_existing: bool = False, ) -> ItemCls: """Add item to library and return the new (or updated) database item.""" new_item = False # check for existing item first - library_id = await self._get_library_item_by_match(item, overwrite_existing) - if library_id is None: + if library_id := await self._get_library_item_by_match(item): + # update existing item + await self._update_library_item(library_id, item, overwrite=overwrite_existing) + else: # actually add a new item in the library db async with self._db_add_lock: library_id = await self._add_library_item(item) new_item = True - # grab additional metadata - if metadata_lookup: - library_item = await self.get_library_item(library_id) - await self.mass.metadata.update_metadata(library_item) - # return final library_item after all match/metadata actions + # return final library_item library_item = await self.get_library_item(library_id) self.mass.signal_event( EventType.MEDIA_ITEM_ADDED if new_item else EventType.MEDIA_ITEM_UPDATED, @@ -116,20 +113,13 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): ) return library_item - async def _get_library_item_by_match( - self, item: Track, overwrite_existing: bool = False - ) -> int | None: + async def _get_library_item_by_match(self, item: Track) -> int | None: if cur_item := await self.get_library_item_by_prov_id(item.item_id, item.provider): - # existing item match by provider id - await self._update_library_item(cur_item.item_id, item, overwrite=overwrite_existing) return cur_item.item_id if cur_item := await self.get_library_item_by_external_ids(item.external_ids): # existing item match by external id # Double check external IDs - if MBID exists, regards that as overriding if compare_media_item(item, cur_item): - await self._update_library_item( - cur_item.item_id, item, overwrite=overwrite_existing - ) return cur_item.item_id # search by (exact) name match query = f"WHERE {self.db_table}.name = :name OR {self.db_table}.sort_name = :sort_name" @@ -138,8 +128,6 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): extra_query=query, extra_query_params=query_params ): if compare_media_item(db_item, item, True): - # existing item found: update it - await self._update_library_item(db_item.item_id, item, overwrite=overwrite_existing) return db_item.item_id return None @@ -656,7 +644,6 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): async def _add_library_item( self, item: ItemCls, - metadata_lookup: bool = True, overwrite_existing: bool = False, ) -> int: """Add artist to library and return the database id.""" diff --git a/music_assistant/server/controllers/media/tracks.py b/music_assistant/server/controllers/media/tracks.py index 42fe6742..25a01372 100644 --- a/music_assistant/server/controllers/media/tracks.py +++ b/music_assistant/server/controllers/media/tracks.py @@ -426,7 +426,6 @@ class TracksController(MediaControllerBase[Track]): with suppress(MediaNotFoundError, AssertionError, InvalidDataError): db_album = await self.mass.music.albums.add_item_to_library( album, - metadata_lookup=False, overwrite_existing=overwrite, ) if not db_album: @@ -487,7 +486,7 @@ class TracksController(MediaControllerBase[Track]): if not db_artist or overwrite: db_artist = await self.mass.music.artists.add_item_to_library( - artist, metadata_lookup=False, overwrite_existing=overwrite + artist, overwrite_existing=overwrite ) # write (or update) record in album_artists table await self.mass.music.database.insert_or_replace( diff --git a/music_assistant/server/controllers/metadata.py b/music_assistant/server/controllers/metadata.py index 0d6ee277..4680143f 100644 --- a/music_assistant/server/controllers/metadata.py +++ b/music_assistant/server/controllers/metadata.py @@ -114,6 +114,7 @@ class MetaDataController(CoreController): ) self.manifest.icon = "book-information-variant" self._reset_online_slots() + self._scanner_running: bool = False async def get_config_entries( self, @@ -146,7 +147,6 @@ class MetaDataController(CoreController): await asyncio.to_thread(os.mkdir, self._collage_images_dir) self.mass.streams.register_dynamic_route("/imageproxy", self.handle_imageproxy) - self.mass.call_later(60, self._metadata_scanner()) async def close(self) -> None: """Handle logic on server stop.""" @@ -224,6 +224,38 @@ class MetaDataController(CoreController): if item.media_type == MediaType.RADIO: await self._update_radio_metadata(item) + @api_command("metadata/scan") + async def metadata_scanner(self) -> None: + """Scanner for (missing) metadata.""" + if self._scanner_running: + # already running + return + self._scanner_running = True + try: + timestamp = int(time() - 60 * 60 * 24 * 7) + query = ( + "WHERE json_extract(metadata,'$.last_refresh') ISNULL " + f"OR json_extract(metadata,'$.last_refresh') < {timestamp}" + ) + for artist in await self.mass.music.artists.library_items( + limit=250, order_by="random", extra_query=query + ): + await self._update_artist_metadata(artist) + for album in await self.mass.music.albums.library_items( + limit=250, order_by="random", extra_query=query + ): + await self._update_album_metadata(album) + for track in await self.mass.music.tracks.library_items( + limit=50, order_by="random", extra_query=query + ): + await self._update_track_metadata(track) + for playlist in await self.mass.music.playlists.library_items( + limit=250, order_by="random", extra_query=query + ): + await self._update_playlist_metadata(playlist) + finally: + self._scanner_running = False + async def get_image_data_for_item( self, media_item: MediaItemType, @@ -399,7 +431,10 @@ class MetaDataController(CoreController): artist.metadata.last_refresh = int(time()) # TODO: Use a global cache/proxy for the MB lookups to save on API calls - artist.mbid = artist.mbid or await self._get_artist_mbid(artist) + if not artist.mbid: + if mbid := await self._get_artist_mbid(artist): + artist.mbid = mbid + if artist.mbid: # The musicbrainz ID is mandatory for all metadata lookups for provider in self.providers: @@ -617,28 +652,3 @@ class MetaDataController(CoreController): self._online_slots_available = MAX_ONLINE_CALLS_PER_DAY # reschedule self in 24 hours self.mass.loop.call_later(60 * 60 * 24, self._reset_online_slots) - - async def _metadata_scanner(self) -> None: - """Continuously (slow) background scanner for (missing) metadata.""" - while True: - for artist in await self.mass.music.artists.library_items(order_by="random"): - if (time() - (artist.metadata.last_refresh or 0)) < REFRESH_INTERVAL: - await asyncio.sleep(5) - continue - await self._update_artist_metadata(artist) - await asyncio.sleep(300) - for album in await self.mass.music.albums.library_items(order_by="random"): - if (time() - (album.metadata.last_refresh or 0)) < REFRESH_INTERVAL: - await asyncio.sleep(5) - continue - await self._update_album_metadata(album) - await asyncio.sleep(300) - for track in await self.mass.music.tracks.library_items(order_by="random"): - if (time() - (track.metadata.last_refresh or 0)) < REFRESH_INTERVAL: - await asyncio.sleep(5) - continue - await self._update_track_metadata(track) - await asyncio.sleep(300) - for playlist in await self.mass.music.playlists.library_items(order_by="random"): - await self._update_playlist_metadata(playlist) - await asyncio.sleep(60) diff --git a/music_assistant/server/controllers/music.py b/music_assistant/server/controllers/music.py index bf2e3cb9..aec36a8d 100644 --- a/music_assistant/server/controllers/music.py +++ b/music_assistant/server/controllers/music.py @@ -480,7 +480,10 @@ class MusicController(CoreController): provider = self.mass.get_provider(item.provider) if provider.library_edit_supported(item.media_type): await provider.library_add(item) - return await ctrl.add_item_to_library(item) + library_item = await ctrl.add_item_to_library(item) + # perform full metadata scan (and provider match) + await self.mass.metadata.update_metadata(library_item) + return library_item async def refresh_items(self, items: list[MediaItemType]) -> None: """Refresh MediaItems to force retrieval of full info and matches. @@ -536,7 +539,9 @@ class MusicController(CoreController): media_item = await ctrl.get_provider_item(item_id, provider, force_refresh=True) # update library item if needed (including refresh of the metadata etc.) if is_library_item: - return await ctrl.add_item_to_library(media_item, metadata_lookup=True) + library_item = await ctrl.add_item_to_library(media_item) + await self.mass.metadata.update_metadata(library_item, force_refresh=True) + return library_item return media_item @@ -718,9 +723,10 @@ class MusicController(CoreController): else: self.logger.info("Sync task for %s completed", provider.name) self.mass.signal_event(EventType.SYNC_TASKS_UPDATED, data=self.in_progress_syncs) - # schedule db cleanup after sync + # schedule db cleanup + metadata scan after sync if not self.in_progress_syncs: self.mass.create_task(self._cleanup_database()) + self.mass.create_task(self.mass.metadata.metadata_scanner()) task.add_done_callback(on_sync_task_done) diff --git a/music_assistant/server/models/music_provider.py b/music_assistant/server/models/music_provider.py index 6ec8a2f2..af9c51e1 100644 --- a/music_assistant/server/models/music_provider.py +++ b/music_assistant/server/models/music_provider.py @@ -416,9 +416,7 @@ class MusicProvider(Provider): # the additional metadata is then lazy retrieved afterwards if self.is_streaming_provider: prov_item.favorite = True - library_item = await controller.add_item_to_library( - prov_item, metadata_lookup=False - ) + library_item = await controller.add_item_to_library(prov_item) elif getattr(library_item, "cache_checksum", None) != getattr( prov_item, "cache_checksum", None ): diff --git a/music_assistant/server/providers/filesystem_local/__init__.py b/music_assistant/server/providers/filesystem_local/__init__.py index 33f02d23..20274f2c 100644 --- a/music_assistant/server/providers/filesystem_local/__init__.py +++ b/music_assistant/server/providers/filesystem_local/__init__.py @@ -9,7 +9,6 @@ import re from typing import TYPE_CHECKING import aiofiles -import cchardet import shortuuid from aiofiles.os import wrap @@ -53,7 +52,6 @@ async def setup( prov = LocalFileSystemProvider(mass, manifest, config) prov.base_path = str(config.get_value(CONF_PATH)) await prov.check_write_access() - mass.call_later(30, prov.migrate_playlists) return prov @@ -211,57 +209,3 @@ class LocalFileSystemProvider(FileSystemProviderBase): abs_path = get_absolute_path(self.base_path, file_path) async with aiofiles.open(abs_path, "wb") as _file: await _file.write(data) - - async def migrate_playlists(self) -> None: - """Migrate Music Assistant filesystem playlists.""" - # Remove this code when 2.0 stable has been released! - # prior to version 2.0.0b137 Music Assistant stored universal playlists - # in the filesystem (root of the music dir, m3u files with uri's) - # that is converted into a universal builtin provider approach in b137 - # so the filesystem is not longer polluted/abused for this. - # this code hunts these playlists, migrates them to the universal provider - # and cleans up the files. - cache_key = f"{self.instance_id}.playlist_migration_done" - if await self.mass.cache.get(cache_key): - return - async for item in self.listdir("", False): - if not item.is_file: - continue - if item.ext != "m3u": - continue - playlist_bytes = b"" - async for chunk in self.read_file_content(item.absolute_path): - playlist_bytes += chunk - encoding_details = await asyncio.to_thread(cchardet.detect, playlist_bytes) - playlist_data = playlist_bytes.decode(encoding_details["encoding"] or "utf-8") - # a (legacy) playlist file created by MA does not have EXTINFO tags and has uri's - if "EXTINF" in playlist_data or "://" not in playlist_data: - continue - all_uris: list[str] = [] - skipped_lines = 0 - for playlist_line in playlist_data.split("\n"): - playlist_line = playlist_line.strip() # noqa: PLW2901 - if not playlist_line: - continue - if "://" not in playlist_line: - skipped_lines += 1 - self.logger.debug("Ignoring line in migration playlist: %s", playlist_line) - all_uris.append(playlist_line) - if skipped_lines > len(all_uris): - self.logger.warning("NOT migrating playlist: %s", item.path) - continue - # create playlist on the builtin provider - new_playlist = await self.mass.music.playlists.create_playlist(item.name, "builtin") - # append existing uri's to the new playlist - await self.mass.music.playlists.add_playlist_tracks(new_playlist.item_id, all_uris) - # remove existing item from the library - if library_item := await self.mass.music.playlists.get_library_item_by_prov_id( - item.path, self.instance_id - ): - await self.mass.music.playlists.remove_item_from_library(library_item.item_id) - # remove old file - await asyncio.to_thread(os.remove, item.absolute_path) - # refresh the playlist so it builds the metadata - await self.mass.music.playlists.add_item_to_library(new_playlist, metadata_lookup=True) - self.logger.info("Migrated playlist %s", item.name) - await self.mass.cache.set(cache_key, True, expiration=365 * 86400) diff --git a/music_assistant/server/providers/filesystem_local/base.py b/music_assistant/server/providers/filesystem_local/base.py index 377d0a28..0a0b6f3b 100644 --- a/music_assistant/server/providers/filesystem_local/base.py +++ b/music_assistant/server/providers/filesystem_local/base.py @@ -341,7 +341,7 @@ class FileSystemProviderBase(MusicProvider): # when they are detected as changed track = await self._parse_track(item) await self.mass.music.tracks.add_item_to_library( - track, metadata_lookup=False, overwrite_existing=prev_checksum is not None + track, overwrite_existing=prev_checksum is not None ) elif item.ext in PLAYLIST_EXTENSIONS: playlist = await self.get_playlist(item.path) @@ -351,7 +351,6 @@ class FileSystemProviderBase(MusicProvider): playlist.favorite = True await self.mass.music.playlists.add_item_to_library( playlist, - metadata_lookup=False, overwrite_existing=prev_checksum is not None, ) except Exception as err: # pylint: disable=broad-except diff --git a/music_assistant/server/providers/filesystem_smb/__init__.py b/music_assistant/server/providers/filesystem_smb/__init__.py index 1e2d5fda..e0b5f730 100644 --- a/music_assistant/server/providers/filesystem_smb/__init__.py +++ b/music_assistant/server/providers/filesystem_smb/__init__.py @@ -52,7 +52,6 @@ async def setup( prov = SMBFileSystemProvider(mass, manifest, config) await prov.handle_async_init() await prov.check_write_access() - mass.call_later(30, prov.migrate_playlists) return prov -- 2.34.1