From: Marcel van der Veldt Date: Wed, 15 Jun 2022 15:50:57 +0000 (+0200) Subject: Filesystem improvements (#367) X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=16a3d21bef80f5f8f7d8328a3a045520f5b4cd8c;p=music-assistant-server.git Filesystem improvements (#367) * save the checksums every 50 processed tracks * overwrite info in db when ids tags are changed * overwrite metdata too * update playlists * delete playlist when deleted --- diff --git a/music_assistant/controllers/music/albums.py b/music_assistant/controllers/music/albums.py index 043f4445..b0b78e3c 100644 --- a/music_assistant/controllers/music/albums.py +++ b/music_assistant/controllers/music/albums.py @@ -109,7 +109,9 @@ class AlbumsController(MediaControllerBase[Album]): return [] return await prov.get_album_tracks(item_id) - async def add_db_item(self, item: Album, db: Optional[Db] = None) -> Album: + async def add_db_item( + self, item: Album, overwrite_existing: bool = False, db: Optional[Db] = None + ) -> Album: """Add a new record to the database.""" assert item.provider_ids, f"Album {item.name} is missing provider id(s)" assert item.artist, f"Album {item.name} is missing artist" @@ -134,7 +136,9 @@ class AlbumsController(MediaControllerBase[Album]): break if cur_item: # update existing - return await self.update_db_item(cur_item.item_id, item, db=db) + return await self.update_db_item( + cur_item.item_id, item, overwrite=overwrite_existing, db=db + ) # insert new item album_artists = await self._get_album_artists(item, cur_item, db=db) @@ -169,13 +173,16 @@ class AlbumsController(MediaControllerBase[Album]): assert item.artist, f"Album {item.name} is missing artist" async with self.mass.database.get_db(db) as db: cur_item = await self.get_db_item(item_id) - album_artists = await self._get_album_artists(item, cur_item, db=db) + if overwrite: metadata = item.metadata + metadata.last_refresh = None provider_ids = item.provider_ids + album_artists = await self._get_album_artists(cur_item, db=db) else: metadata = cur_item.metadata.update(item.metadata) provider_ids = {*cur_item.provider_ids, *item.provider_ids} + album_artists = await self._get_album_artists(item, cur_item, db=db) if item.album_type != AlbumType.UNKNOWN: album_type = item.album_type diff --git a/music_assistant/controllers/music/artists.py b/music_assistant/controllers/music/artists.py index 1476153d..3be1cf4a 100644 --- a/music_assistant/controllers/music/artists.py +++ b/music_assistant/controllers/music/artists.py @@ -131,7 +131,9 @@ class ArtistsController(MediaControllerBase[Artist]): return [] return await provider.get_artist_albums(item_id) - async def add_db_item(self, item: Artist, db: Optional[Db] = None) -> Artist: + async def add_db_item( + self, item: Artist, overwrite_existing: bool = False, db: Optional[Db] = None + ) -> Artist: """Add a new item record to the database.""" assert item.provider_ids, "Album is missing provider id(s)" async with self.mass.database.get_db(db) as db: @@ -156,7 +158,9 @@ class ArtistsController(MediaControllerBase[Artist]): break if cur_item: # update existing - return await self.update_db_item(cur_item.item_id, item, db=db) + return await self.update_db_item( + cur_item.item_id, item, overwrite=overwrite_existing, db=db + ) # insert item new_item = await self.mass.database.insert( diff --git a/music_assistant/controllers/music/playlists.py b/music_assistant/controllers/music/playlists.py index 66c1a388..eeca5a4e 100644 --- a/music_assistant/controllers/music/playlists.py +++ b/music_assistant/controllers/music/playlists.py @@ -162,7 +162,9 @@ class PlaylistController(MediaControllerBase[Playlist]): ) ) - async def add_db_item(self, item: Playlist, db: Optional[Db] = None) -> Playlist: + async def add_db_item( + self, item: Playlist, overwrite_existing: bool = False, db: Optional[Db] = None + ) -> Playlist: """Add a new record to the database.""" async with self.mass.database.get_db(db) as db: match = {"name": item.name, "owner": item.owner} @@ -170,7 +172,9 @@ class PlaylistController(MediaControllerBase[Playlist]): self.db_table, match, db=db ): # update existing - return await self.update_db_item(cur_item["item_id"], item, db=db) + return await self.update_db_item( + cur_item["item_id"], item, overwrite=overwrite_existing, db=db + ) # insert new item new_item = await self.mass.database.insert( diff --git a/music_assistant/controllers/music/radio.py b/music_assistant/controllers/music/radio.py index 1e3962af..5ac866ab 100644 --- a/music_assistant/controllers/music/radio.py +++ b/music_assistant/controllers/music/radio.py @@ -31,7 +31,9 @@ class RadioController(MediaControllerBase[Radio]): await self.mass.metadata.get_radio_metadata(item) return await self.add_db_item(item) - async def add_db_item(self, item: Radio, db: Optional[Db] = None) -> Radio: + async def add_db_item( + self, item: Radio, overwrite_existing: bool = False, db: Optional[Db] = None + ) -> Radio: """Add a new item record to the database.""" assert item.provider_ids async with self.mass.database.get_db(db) as db: @@ -40,7 +42,9 @@ class RadioController(MediaControllerBase[Radio]): self.db_table, match, db=db ): # update existing - return await self.update_db_item(cur_item["item_id"], item, db=db) + return await self.update_db_item( + cur_item["item_id"], item, overwrite=overwrite_existing, db=db + ) # insert new item new_item = await self.mass.database.insert( diff --git a/music_assistant/controllers/music/tracks.py b/music_assistant/controllers/music/tracks.py index fb8c018f..01fa38f2 100644 --- a/music_assistant/controllers/music/tracks.py +++ b/music_assistant/controllers/music/tracks.py @@ -115,7 +115,9 @@ class TracksController(MediaControllerBase[Track]): provider.name, ) - async def add_db_item(self, item: Track, db: Optional[Db] = None) -> Track: + async def add_db_item( + self, item: Track, overwrite_existing: bool = False, db: Optional[Db] = None + ) -> Track: """Add a new item record to the database.""" assert item.artists, "Track is missing artist(s)" assert item.provider_ids, "Track is missing provider id(s)" @@ -141,7 +143,9 @@ class TracksController(MediaControllerBase[Track]): break if cur_item: # update existing - return await self.update_db_item(cur_item.item_id, item, db=db) + return await self.update_db_item( + cur_item.item_id, item, overwrite=overwrite_existing, db=db + ) # no existing match found: insert new item track_artists = await self._get_track_artists(item, db=db) @@ -176,15 +180,20 @@ class TracksController(MediaControllerBase[Track]): """Update Track record in the database, merging data.""" async with self.mass.database.get_db(db) as db: cur_item = await self.get_db_item(item_id, db=db) + if overwrite: + metadata = item.metadata provider_ids = item.provider_ids + metadata.last_refresh = None + # we store a mapping to artists/albums on the item for easier access/listings + track_artists = await self._get_track_artists(item, db=db) + track_albums = await self._get_track_albums(item, db=db) else: + metadata = cur_item.metadata.update(item.metadata, overwrite) provider_ids = {*cur_item.provider_ids, *item.provider_ids} - metadata = cur_item.metadata.update(item.metadata, overwrite) + track_artists = await self._get_track_artists(cur_item, item, db=db) + track_albums = await self._get_track_albums(cur_item, item, db=db) - # we store a mapping to artists/albums on the item for easier access/listings - track_artists = await self._get_track_artists(cur_item, item, db=db) - track_albums = await self._get_track_albums(cur_item, item, db=db) await self.mass.database.update( self.db_table, {"item_id": item_id}, diff --git a/music_assistant/models/media_controller.py b/music_assistant/models/media_controller.py index babf4cf6..08bffa3a 100644 --- a/music_assistant/models/media_controller.py +++ b/music_assistant/models/media_controller.py @@ -47,7 +47,9 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): raise NotImplementedError @abstractmethod - async def add_db_item(self, item: ItemCls, db: Optional[Db] = None) -> ItemCls: + async def add_db_item( + self, item: ItemCls, overwrite_existing: bool = False, db: Optional[Db] = None + ) -> ItemCls: """Add a new record for this mediatype to the database.""" raise NotImplementedError diff --git a/music_assistant/music_providers/filesystem.py b/music_assistant/music_providers/filesystem.py index 6658cb38..3334dd52 100644 --- a/music_assistant/music_providers/filesystem.py +++ b/music_assistant/music_providers/filesystem.py @@ -16,7 +16,6 @@ from tinytag.tinytag import TinyTag from music_assistant.helpers.audio import get_file_stream from music_assistant.helpers.compare import compare_strings -from music_assistant.helpers.database import SCHEMA_VERSION from music_assistant.helpers.util import ( create_clean_string, parse_title_and_version, @@ -53,6 +52,7 @@ CONTENT_TYPE_EXT = { "ogg": ContentType.OGG, "wma": ContentType.WMA, } +SCHEMA_VERSION = 17 async def scantree(path: str) -> AsyncGenerator[os.DirEntry, None]: @@ -145,6 +145,7 @@ class FileSystemProvider(MusicProvider): """Run library sync for this provider.""" cache_key = f"{self.id}.checksums" prev_checksums = await self.mass.cache.get(cache_key, SCHEMA_VERSION) + save_checksum_interval = 0 if prev_checksums is None: prev_checksums = {} # find all music files in the music directory and all subfolders @@ -166,7 +167,7 @@ class FileSystemProvider(MusicProvider): # process album if track.album: db_album = await self.mass.music.albums.add_db_item( - track.album, db=db + track.album, allow_overwrite=True, db=db ) if not db_album.in_library: await self.mass.music.albums.set_db_library( @@ -183,7 +184,7 @@ class FileSystemProvider(MusicProvider): ) # add/update track to db db_track = await self.mass.music.tracks.add_db_item( - track, db=db + track, allow_overwrite=True, db=db ) if not db_track.in_library: await self.mass.music.tracks.set_db_library( @@ -197,16 +198,23 @@ class FileSystemProvider(MusicProvider): # we don't want the whole sync to crash on one file so we catch all exceptions here self.logger.exception("Error processing %s", entry.path) - # save checksums for next sync - await self.mass.cache.set(cache_key, cur_checksums, SCHEMA_VERSION) + # save checksums every 50 processed items + # this allows us to pickup where we leftoff when initial scan gets intterrupted + if save_checksum_interval == 50: + await self.mass.cache.set(cache_key, cur_checksums, SCHEMA_VERSION) + save_checksum_interval = 0 + else: + save_checksum_interval += 1 + await self.mass.cache.set(cache_key, cur_checksums, SCHEMA_VERSION) # work out deletions deleted_files = set(prev_checksums.keys()) - set(cur_checksums.keys()) artists: Set[ItemMapping] = set() albums: Set[ItemMapping] = set() - # process deleted tracks + # process deleted tracks/playlists for file_path in deleted_files: item_id = self._get_item_id(file_path) + # try track first if db_item := await self.mass.music.tracks.get_db_item_by_prov_id( item_id, self.type ): @@ -221,6 +229,13 @@ class FileSystemProvider(MusicProvider): albums.add(db_item.album.item_id) for artist in db_item.album.artists: artists.add(artist.item_id) + # fallback to playlist + elif db_item := await self.mass.music.playlists.get_db_item_by_prov_id( + item_id, self.type + ): + await self.mass.music.playlists.remove_prov_mapping( + db_item.item_id, self.id + ) # check if albums are deleted for album_id in albums: album = await self.mass.music.albums.get_db_item(album_id)