From: Marcel van der Veldt Date: Thu, 19 May 2022 12:41:07 +0000 (+0200) Subject: Optimize database transactions (#327) X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=2d2224c4323297e29cc88f5da2eda1070968e320;p=music-assistant-server.git Optimize database transactions (#327) --- diff --git a/music_assistant/controllers/metadata/__init__.py b/music_assistant/controllers/metadata/__init__.py index ff1b8e0c..b68a540b 100755 --- a/music_assistant/controllers/metadata/__init__.py +++ b/music_assistant/controllers/metadata/__init__.py @@ -233,8 +233,8 @@ class MetaDataController: else: # create thumbnail if it doesn't exist thumbnail = await create_thumbnail(self.mass, path, size) - await self.mass.database.insert_or_replace( - TABLE_THUMBS, {**match, "data": thumbnail} + await self.mass.database.insert( + TABLE_THUMBS, {**match, "data": thumbnail}, allow_replace=True ) if base64: enc_image = b64encode(thumbnail).decode() diff --git a/music_assistant/controllers/music/__init__.py b/music_assistant/controllers/music/__init__.py index f673d77d..2d38e251 100755 --- a/music_assistant/controllers/music/__init__.py +++ b/music_assistant/controllers/music/__init__.py @@ -282,15 +282,15 @@ class MusicController: db: Optional[Db] = None, ): """Store provider ids for media item to database.""" - async with self.mass.database.get_db(db) as _db: + async with self.mass.database.get_db(db) as db: # make sure that existing items are deleted first await self.mass.database.delete( TABLE_PROV_MAPPINGS, {"item_id": int(item_id), "media_type": media_type.value}, - db=_db, + db=db, ) for prov_id in prov_ids: - await self.mass.database.insert_or_replace( + await self.mass.database.insert( TABLE_PROV_MAPPINGS, { "item_id": item_id, @@ -302,7 +302,8 @@ class MusicController: "details": prov_id.details, "url": prov_id.url, }, - db=_db, + allow_replace=True, + db=db, ) async def refresh_items(self, items: List[MediaItem]) -> None: @@ -341,9 +342,10 @@ class MusicController: self, item_id: str, provider: ProviderType, loudness: int ): """List integrated loudness for a track in db.""" - await self.mass.database.insert_or_replace( + await self.mass.database.insert( TABLE_TRACK_LOUDNESS, {"item_id": item_id, "provider": provider.value, "loudness": loudness}, + allow_replace=True, ) async def get_track_loudness( @@ -377,9 +379,10 @@ class MusicController: async def mark_item_played(self, item_id: str, provider: ProviderType): """Mark item as played in playlog.""" timestamp = utc_timestamp() - await self.mass.database.insert_or_replace( + await self.mass.database.insert( TABLE_PLAYLOG, {"item_id": item_id, "provider": provider.value, "timestamp": timestamp}, + allow_replace=True, ) async def library_add_items(self, items: List[MediaItem]) -> None: diff --git a/music_assistant/controllers/music/albums.py b/music_assistant/controllers/music/albums.py index 6471aa65..ac55b90d 100644 --- a/music_assistant/controllers/music/albums.py +++ b/music_assistant/controllers/music/albums.py @@ -5,6 +5,8 @@ import asyncio import itertools from typing import Dict, List, Optional +from databases import Database as Db + from music_assistant.helpers.compare import compare_album, compare_artist from music_assistant.helpers.database import TABLE_ALBUMS from music_assistant.helpers.json import json_serializer @@ -110,27 +112,23 @@ class AlbumsController(MediaControllerBase[Album]): return [] return await prov.get_album_tracks(item_id) - async def add_db_item(self, album: Album) -> Album: + async def add_db_item(self, album: Album, db: Optional[Db] = None) -> Album: """Add a new album record to the database.""" assert album.provider_ids, "Album is missing provider id(s)" cur_item = None - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: # always try to grab existing item by musicbrainz_id if album.musicbrainz_id: match = {"musicbrainz_id": album.musicbrainz_id} - cur_item = await self.mass.database.get_row( - self.db_table, match, db=_db - ) + cur_item = await self.mass.database.get_row(self.db_table, match, db=db) if not cur_item and album.upc: match = {"upc": album.upc} - cur_item = await self.mass.database.get_row( - self.db_table, match, db=_db - ) + cur_item = await self.mass.database.get_row(self.db_table, match, db=db) if not cur_item: # fallback to matching match = {"sort_name": album.sort_name} for row in await self.mass.database.get_rows( - self.db_table, match, db=_db + self.db_table, match, db=db ): row_album = Album.from_db_row(row) if compare_album(row_album, album): @@ -138,34 +136,38 @@ class AlbumsController(MediaControllerBase[Album]): break if cur_item: # update existing - return await self.update_db_item(cur_item.item_id, album) + return await self.update_db_item(cur_item.item_id, album, db=db) # insert new album - album_artist = await self._get_album_artist(album, cur_item) - new_item = await self.mass.database.insert_or_replace( + album_artist = await self._get_album_artist(album, cur_item, db=db) + new_item = await self.mass.database.insert( self.db_table, { **album.to_db_row(), "artist": json_serializer(album_artist) or None, }, - db=_db, + db=db, ) item_id = new_item["item_id"] # store provider mappings await self.mass.music.set_provider_mappings( - item_id, MediaType.ALBUM, album.provider_ids, db=_db + item_id, MediaType.ALBUM, album.provider_ids, db=db ) self.logger.debug("added %s to database", album.name) # return created object - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def update_db_item( - self, item_id: int, album: Album, overwrite: bool = False + self, + item_id: int, + album: Album, + overwrite: bool = False, + db: Optional[Db] = None, ) -> Album: """Update Album record in the database.""" - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: cur_item = await self.get_db_item(item_id) - album_artist = await self._get_album_artist(album, cur_item) + album_artist = await self._get_album_artist(album, cur_item, db=db) if overwrite: metadata = album.metadata provider_ids = album.provider_ids @@ -192,13 +194,13 @@ class AlbumsController(MediaControllerBase[Album]): "metadata": json_serializer(metadata), "provider_ids": json_serializer(provider_ids), }, - db=_db, + db=db, ) await self.mass.music.set_provider_mappings( - item_id, MediaType.ALBUM, provider_ids, db=_db + item_id, MediaType.ALBUM, provider_ids, db=db ) self.logger.debug("updated %s in database: %s", album.name, item_id) - return await self.get_db_item(item_id) + return await self.get_db_item(item_id, db=db) async def _match(self, db_album: Album) -> None: """ @@ -247,7 +249,10 @@ class AlbumsController(MediaControllerBase[Album]): ) async def _get_album_artist( - self, db_album: Album, updated_album: Optional[Album] = None + self, + db_album: Album, + updated_album: Optional[Album] = None, + db: Optional[Db] = None, ) -> ItemMapping | None: """Extract (database) album artist as ItemMapping.""" for album in (updated_album, db_album): @@ -260,12 +265,11 @@ class AlbumsController(MediaControllerBase[Album]): return ItemMapping.from_item(album.artist) if db_artist := await self.mass.music.artists.get_db_item_by_prov_id( - album.artist.item_id, - provider=album.artist.provider, + album.artist.item_id, provider=album.artist.provider, db=db ): return ItemMapping.from_item(db_artist) - db_artist = await self.mass.music.artists.add_db_item(album.artist) + db_artist = await self.mass.music.artists.add_db_item(album.artist, db=db) return ItemMapping.from_item(db_artist) return None diff --git a/music_assistant/controllers/music/artists.py b/music_assistant/controllers/music/artists.py index 854c3ed9..d157af19 100644 --- a/music_assistant/controllers/music/artists.py +++ b/music_assistant/controllers/music/artists.py @@ -4,6 +4,8 @@ import asyncio import itertools from typing import List, Optional +from databases import Database as Db + from music_assistant.helpers.compare import ( compare_album, compare_artist, @@ -120,17 +122,15 @@ class ArtistsController(MediaControllerBase[Artist]): return [] return await provider.get_artist_albums(item_id) - async def add_db_item(self, artist: Artist) -> Artist: + async def add_db_item(self, artist: Artist, db: Optional[Db] = None) -> Artist: """Add a new artist record to the database.""" assert artist.provider_ids, "Album is missing provider id(s)" - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: # always try to grab existing item by musicbrainz_id cur_item = None if artist.musicbrainz_id: match = {"musicbrainz_id": artist.musicbrainz_id} - cur_item = await self.mass.database.get_row( - self.db_table, match, db=_db - ) + cur_item = await self.mass.database.get_row(self.db_table, match, db=db) if not cur_item: # fallback to matching # NOTE: we match an artist by name which could theoretically lead to collisions @@ -138,7 +138,7 @@ class ArtistsController(MediaControllerBase[Artist]): # the musicbrainz id upfront match = {"sort_name": artist.sort_name} for row in await self.mass.database.get_rows( - self.db_table, match, db=_db + self.db_table, match, db=db ): row_artist = Artist.from_db_row(row) if compare_strings(row_artist.sort_name, artist.sort_name): @@ -146,23 +146,27 @@ class ArtistsController(MediaControllerBase[Artist]): break if cur_item: # update existing - return await self.update_db_item(cur_item.item_id, artist) + return await self.update_db_item(cur_item.item_id, artist, db=db) # insert artist - new_item = await self.mass.database.insert_or_replace( - self.db_table, artist.to_db_row(), db=_db + new_item = await self.mass.database.insert( + self.db_table, artist.to_db_row(), db=db ) item_id = new_item["item_id"] # store provider mappings await self.mass.music.set_provider_mappings( - item_id, MediaType.ARTIST, artist.provider_ids, db=_db + item_id, MediaType.ARTIST, artist.provider_ids, db=db ) self.logger.debug("added %s to database", artist.name) # return created object - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def update_db_item( - self, item_id: int, artist: Artist, overwrite: bool = False + self, + item_id: int, + artist: Artist, + overwrite: bool = False, + db: Optional[Db] = None, ) -> Artist: """Update Artist record in the database.""" cur_item = await self.get_db_item(item_id) @@ -173,7 +177,7 @@ class ArtistsController(MediaControllerBase[Artist]): metadata = cur_item.metadata.update(artist.metadata) provider_ids = {*cur_item.provider_ids, *artist.provider_ids} - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: await self.mass.database.update( self.db_table, {"item_id": item_id}, @@ -184,13 +188,13 @@ class ArtistsController(MediaControllerBase[Artist]): "metadata": json_serializer(metadata), "provider_ids": json_serializer(provider_ids), }, - db=_db, + db=db, ) await self.mass.music.set_provider_mappings( - item_id, MediaType.ARTIST, provider_ids, db=_db + item_id, MediaType.ARTIST, provider_ids, db=db ) self.logger.debug("updated %s in database: %s", artist.name, item_id) - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def _match(self, db_artist: Artist, provider: MusicProvider) -> bool: """Try to find matching artists on given provider for the provided (database) artist.""" diff --git a/music_assistant/controllers/music/playlists.py b/music_assistant/controllers/music/playlists.py index 12b12871..d9a94713 100644 --- a/music_assistant/controllers/music/playlists.py +++ b/music_assistant/controllers/music/playlists.py @@ -4,6 +4,8 @@ from __future__ import annotations from time import time from typing import List, Optional +from databases import Database as Db + from music_assistant.helpers.database import TABLE_PLAYLISTS from music_assistant.helpers.json import json_serializer from music_assistant.helpers.uri import create_uri @@ -160,42 +162,49 @@ class PlaylistController(MediaControllerBase[Playlist]): ) ) - async def add_db_item(self, playlist: Playlist) -> Playlist: + async def add_db_item( + self, playlist: Playlist, db: Optional[Db] = None + ) -> Playlist: """Add a new playlist record to the database.""" - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: match = {"name": playlist.name, "owner": playlist.owner} if cur_item := await self.mass.database.get_row( - self.db_table, match, db=_db + self.db_table, match, db=db ): # update existing - return await self.update_db_item(cur_item["item_id"], playlist) + return await self.update_db_item(cur_item["item_id"], playlist, db=db) # insert new playlist - new_item = await self.mass.database.insert_or_replace( - self.db_table, playlist.to_db_row(), db=_db + new_item = await self.mass.database.insert( + self.db_table, playlist.to_db_row(), db=db ) item_id = new_item["item_id"] # store provider mappings await self.mass.music.set_provider_mappings( - item_id, MediaType.PLAYLIST, playlist.provider_ids, db=_db + item_id, MediaType.PLAYLIST, playlist.provider_ids, db=db ) self.logger.debug("added %s to database", playlist.name) # return created object - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def update_db_item( - self, item_id: int, playlist: Playlist, overwrite: bool = False + self, + item_id: int, + playlist: Playlist, + overwrite: bool = False, + db: Optional[Db] = None, ) -> Playlist: """Update Playlist record in the database.""" - cur_item = await self.get_db_item(item_id) - if overwrite: - metadata = playlist.metadata - provider_ids = playlist.provider_ids - else: - metadata = cur_item.metadata.update(playlist.metadata) - provider_ids = {*cur_item.provider_ids, *playlist.provider_ids} - - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: + + cur_item = await self.get_db_item(item_id, db=db) + if overwrite: + metadata = playlist.metadata + provider_ids = playlist.provider_ids + else: + metadata = cur_item.metadata.update(playlist.metadata) + provider_ids = {*cur_item.provider_ids, *playlist.provider_ids} + await self.mass.database.update( self.db_table, {"item_id": item_id}, @@ -207,13 +216,13 @@ class PlaylistController(MediaControllerBase[Playlist]): "metadata": json_serializer(metadata), "provider_ids": json_serializer(provider_ids), }, - db=_db, + db=db, ) await self.mass.music.set_provider_mappings( - item_id, MediaType.PLAYLIST, provider_ids, db=_db + item_id, MediaType.PLAYLIST, provider_ids, db=db ) self.logger.debug("updated %s in database: %s", playlist.name, item_id) - db_item = await self.get_db_item(item_id, db=_db) + db_item = await self.get_db_item(item_id, db=db) self.mass.signal_event( MassEvent( type=EventType.PLAYLIST_UPDATED, object_id=item_id, data=playlist diff --git a/music_assistant/controllers/music/radio.py b/music_assistant/controllers/music/radio.py index 2f1166fc..6c3405ba 100644 --- a/music_assistant/controllers/music/radio.py +++ b/music_assistant/controllers/music/radio.py @@ -2,6 +2,9 @@ from __future__ import annotations from time import time +from typing import Optional + +from databases import Database as Db from music_assistant.helpers.database import TABLE_RADIOS from music_assistant.helpers.json import json_serializer @@ -32,36 +35,40 @@ class RadioController(MediaControllerBase[Radio]): ) return db_item - async def add_db_item(self, radio: Radio) -> Radio: + async def add_db_item(self, radio: Radio, db: Optional[Db] = None) -> Radio: """Add a new radio record to the database.""" assert radio.provider_ids - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: match = {"name": radio.name} if cur_item := await self.mass.database.get_row( - self.db_table, match, db=_db + self.db_table, match, db=db ): # update existing - return await self.update_db_item(cur_item["item_id"], radio) + return await self.update_db_item(cur_item["item_id"], radio, db=db) # insert new radio - new_item = await self.mass.database.insert_or_replace( - self.db_table, radio.to_db_row(), db=_db + new_item = await self.mass.database.insert( + self.db_table, radio.to_db_row(), db=db ) item_id = new_item["item_id"] # store provider mappings await self.mass.music.set_provider_mappings( - item_id, MediaType.RADIO, radio.provider_ids, db=_db + item_id, MediaType.RADIO, radio.provider_ids, db=db ) self.logger.debug("added %s to database", radio.name) # return created object - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def update_db_item( - self, item_id: int, radio: Radio, overwrite: bool = False + self, + item_id: int, + radio: Radio, + overwrite: bool = False, + db: Optional[Db] = None, ) -> Radio: """Update Radio record in the database.""" - async with self.mass.database.get_db() as _db: - cur_item = await self.get_db_item(item_id, db=_db) + async with self.mass.database.get_db(db) as db: + cur_item = await self.get_db_item(item_id, db=db) if overwrite: metadata = radio.metadata provider_ids = radio.provider_ids @@ -79,10 +86,10 @@ class RadioController(MediaControllerBase[Radio]): "metadata": json_serializer(metadata), "provider_ids": json_serializer(provider_ids), }, - db=_db, + db=db, ) await self.mass.music.set_provider_mappings( - item_id, MediaType.RADIO, provider_ids, db=_db + item_id, MediaType.RADIO, provider_ids, db=db ) self.logger.debug("updated %s in database: %s", radio.name, item_id) - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) diff --git a/music_assistant/controllers/music/tracks.py b/music_assistant/controllers/music/tracks.py index 3e586a43..b7a476bb 100644 --- a/music_assistant/controllers/music/tracks.py +++ b/music_assistant/controllers/music/tracks.py @@ -4,6 +4,8 @@ from __future__ import annotations import asyncio from typing import List, Optional, Union +from databases import Database as Db + from music_assistant.helpers.compare import compare_artists, compare_track from music_assistant.helpers.database import TABLE_TRACKS from music_assistant.helpers.json import json_serializer @@ -104,30 +106,26 @@ class TracksController(MediaControllerBase[Track]): provider.name, ) - async def add_db_item(self, track: Track) -> Track: + async def add_db_item(self, track: Track, db: Optional[Db] = None) -> Track: """Add a new track record to the database.""" assert track.artists, "Track is missing artist(s)" assert track.provider_ids, "Track is missing provider id(s)" cur_item = None - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: track_album = await self._get_track_album(track) # always try to grab existing item by external_id if track.musicbrainz_id: match = {"musicbrainz_id": track.musicbrainz_id} - cur_item = await self.mass.database.get_row( - self.db_table, match, db=_db - ) + cur_item = await self.mass.database.get_row(self.db_table, match, db=db) if not cur_item and track.isrc: match = {"isrc": track.isrc} - cur_item = await self.mass.database.get_row( - self.db_table, match, db=_db - ) + cur_item = await self.mass.database.get_row(self.db_table, match, db=db) if not cur_item: # fallback to matching match = {"sort_name": track.sort_name} for row in await self.mass.database.get_rows( - self.db_table, match, db=_db + self.db_table, match, db=db ): row_track = Track.from_db_row(row) if compare_track(row_track, track): @@ -135,35 +133,39 @@ class TracksController(MediaControllerBase[Track]): break if cur_item: # update existing - return await self.update_db_item(cur_item.item_id, track) + return await self.update_db_item(cur_item.item_id, track, db=db) # no existing match found: insert new track - track_artists = await self._get_track_artists(track) - new_item = await self.mass.database.insert_or_replace( + track_artists = await self._get_track_artists(track, db=db) + new_item = await self.mass.database.insert( self.db_table, { **track.to_db_row(), "artists": json_serializer(track_artists), "album": json_serializer(track_album) or None, }, - db=_db, + db=db, ) item_id = new_item["item_id"] # store provider mappings await self.mass.music.set_provider_mappings( - item_id, MediaType.TRACK, track.provider_ids, db=_db + item_id, MediaType.TRACK, track.provider_ids, db=db ) # return created object self.logger.debug("added %s to database: %s", track.name, item_id) - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def update_db_item( - self, item_id: int, track: Track, overwrite: bool = False + self, + item_id: int, + track: Track, + overwrite: bool = False, + db: Optional[Db] = None, ) -> Track: """Update Track record in the database, merging data.""" - async with self.mass.database.get_db() as _db: - cur_item = await self.get_db_item(item_id, db=_db) + async with self.mass.database.get_db(db) as db: + cur_item = await self.get_db_item(item_id, db=db) track_album = await self._get_track_album(cur_item, track) if overwrite: provider_ids = track.provider_ids @@ -189,17 +191,20 @@ class TracksController(MediaControllerBase[Track]): "disc_number": track.disc_number or cur_item.disc_number, "track_number": track.track_number or cur_item.track_number, }, - db=_db, + db=db, ) await self.mass.music.set_provider_mappings( - item_id, MediaType.TRACK, provider_ids, db=_db + item_id, MediaType.TRACK, provider_ids, db=db ) self.logger.debug("updated %s in database: %s", track.name, item_id) - return await self.get_db_item(item_id, db=_db) + return await self.get_db_item(item_id, db=db) async def _get_track_artists( - self, base_track: Track, upd_track: Optional[Track] = None + self, + base_track: Track, + upd_track: Optional[Track] = None, + db: Optional[Db] = None, ) -> List[ItemMapping]: """Extract all (unique) artists of track as ItemMapping.""" if upd_track and upd_track.artists: @@ -207,10 +212,13 @@ class TracksController(MediaControllerBase[Track]): else: track_artists = base_track.artists # use intermediate set to clear out duplicates - return list({await self._get_artist_mapping(x) for x in track_artists}) + return list({await self._get_artist_mapping(x, db=db) for x in track_artists}) async def _get_track_album( - self, base_track: Track, upd_track: Optional[Track] = None + self, + base_track: Track, + upd_track: Optional[Track] = None, + db: Optional[Db] = None, ) -> ItemMapping | None: """Extract (database) track album as ItemMapping.""" for track in (upd_track, base_track): @@ -223,18 +231,17 @@ class TracksController(MediaControllerBase[Track]): return ItemMapping.from_item(track.album) if db_album := await self.mass.music.albums.get_db_item_by_prov_id( - track.album.item_id, - provider=track.album.provider, + track.album.item_id, provider=track.album.provider, db=db ): return ItemMapping.from_item(db_album) - db_album = await self.mass.music.albums.add_db_item(track.album) + db_album = await self.mass.music.albums.add_db_item(track.album, db=db) return ItemMapping.from_item(db_album) return None async def _get_artist_mapping( - self, artist: Union[Artist, ItemMapping] + self, artist: Union[Artist, ItemMapping], db: Optional[Db] = None ) -> ItemMapping: """Extract (database) track artist as ItemMapping.""" if artist.provider == ProviderType.DATABASE: @@ -243,10 +250,9 @@ class TracksController(MediaControllerBase[Track]): return ItemMapping.from_item(artist) if db_artist := await self.mass.music.artists.get_db_item_by_prov_id( - artist.item_id, - provider=artist.provider, + artist.item_id, provider=artist.provider, db=db ): return ItemMapping.from_item(db_artist) - db_artist = await self.mass.music.artists.add_db_item(artist) + db_artist = await self.mass.music.artists.add_db_item(artist, db=db) return ItemMapping.from_item(db_artist) diff --git a/music_assistant/helpers/cache.py b/music_assistant/helpers/cache.py index 815b7285..c817c34b 100644 --- a/music_assistant/helpers/cache.py +++ b/music_assistant/helpers/cache.py @@ -83,9 +83,10 @@ class Cache: # do not cache items in db with short expiration return data = await asyncio.get_running_loop().run_in_executor(None, json.dumps, data) - await self.mass.database.insert_or_replace( + await self.mass.database.insert( TABLE_CACHE, {"key": cache_key, "expires": expires, "checksum": checksum, "data": data}, + allow_replace=True, ) async def delete(self, cache_key): diff --git a/music_assistant/helpers/database.py b/music_assistant/helpers/database.py index c73cdac8..a3b381ff 100755 --- a/music_assistant/helpers/database.py +++ b/music_assistant/helpers/database.py @@ -62,8 +62,8 @@ class Database: """Set setting in settings table.""" if not isinstance(value, str): value = str(value) - return await self.insert_or_replace( - TABLE_SETTINGS, {"key": key, "value": value} + return await self.insert( + TABLE_SETTINGS, {"key": key, "value": value}, allow_replace=True ) async def get_count( @@ -122,13 +122,20 @@ class Database: sql_query += " AND ".join((f"{x} = :{x}" for x in match)) return await _db.fetch_one(sql_query, match) - async def insert_or_replace( - self, table: str, values: Dict[str, Any], db: Optional[Db] = None + async def insert( + self, + table: str, + values: Dict[str, Any], + allow_replace: bool = False, + db: Optional[Db] = None, ) -> Mapping: - """Insert or replace data in given table.""" + """Insert data in given table.""" async with self.get_db(db) as _db: keys = tuple(values.keys()) - sql_query = f'INSERT OR REPLACE INTO {table}({",".join(keys)})' + if allow_replace: + sql_query = f'INSERT OR REPLACE INTO {table}({",".join(keys)})' + else: + sql_query = f'INSERT INTO {table}({",".join(keys)})' sql_query += f' VALUES ({",".join((f":{x}" for x in keys))})' await _db.execute(sql_query, values) # return inserted/replaced item @@ -139,6 +146,12 @@ class Database: } return await self.get_row(table, lookup_vals, db=_db) + async def insert_or_replace( + self, table: str, values: Dict[str, Any], db: Optional[Db] = None + ) -> Mapping: + """Insert or replace data in given table.""" + return await self.insert(table=table, values=values, allow_replace=True, db=db) + async def update( self, table: str, diff --git a/music_assistant/models/media_controller.py b/music_assistant/models/media_controller.py index bc3b97ca..10181eb7 100644 --- a/music_assistant/models/media_controller.py +++ b/music_assistant/models/media_controller.py @@ -161,11 +161,13 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): if db_item.in_library: await self.set_db_library(db_item.item_id, False) - async def get_provider_id(self, item: ItemCls) -> Tuple[str, str]: + async def get_provider_id( + self, item: ItemCls, db: Optional[Db] = None + ) -> Tuple[str, str]: """Return provider and item id.""" if item.provider == ProviderType.DATABASE: # make sure we have a full object - item = await self.get_db_item(item.item_id) + item = await self.get_db_item(item.item_id, db=db) for prov in item.provider_ids: # returns the first provider that is available if not prov.available: @@ -225,13 +227,13 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): query = f"SELECT * FROM tracks WHERE item_id in {str(tuple(db_ids))}" return await self.get_db_items(query, db=db) - async def set_db_library(self, item_id: int, in_library: bool) -> None: + async def set_db_library( + self, item_id: int, in_library: bool, db: Optional[Db] = None + ) -> None: """Set the in-library bool on a database item.""" match = {"item_id": item_id} await self.mass.database.update( - self.db_table, - match, - {"in_library": in_library}, + self.db_table, match, {"in_library": in_library}, db=db ) async def get_provider_item( @@ -255,21 +257,21 @@ class MediaControllerBase(Generic[ItemCls], metaclass=ABCMeta): ) return item - async def delete_db_item(self, item_id: int) -> None: + async def delete_db_item(self, item_id: int, db: Optional[Db] = None) -> None: """Delete record from the database.""" - async with self.mass.database.get_db() as _db: + async with self.mass.database.get_db(db) as db: # delete prov mappings await self.mass.database.delete( TABLE_PROV_MAPPINGS, {"item_id": int(item_id), "media_type": self.media_type.value}, - db=_db, + db=db, ) # delete item await self.mass.database.delete( self.db_table, {"item_id": int(item_id)}, - db=_db, + db=db, ) # NOTE: this does not delete any references to this item in other records self.logger.debug("deleted item with id %s from database", item_id) diff --git a/music_assistant/models/provider.py b/music_assistant/models/provider.py index 1b9c40a1..7088b19c 100644 --- a/music_assistant/models/provider.py +++ b/music_assistant/models/provider.py @@ -193,44 +193,45 @@ class MusicProvider: # this logic is aimed at streaming/online providers, # which all have more or less the same structure. # filesystem implementation(s) just override this. - for media_type in self.supported_mediatypes: - - self.logger.debug("Start sync of %s items.", media_type.value) - controller = self.mass.music.get_controller(media_type) - - # create a set of all previous and current db id's - prev_ids = set() - for db_item in await controller.library(): - for prov_id in db_item.provider_ids: - if prov_id.prov_id == self.id: - prev_ids.add(db_item.item_id) - cur_ids = set() - async for prov_item in self._get_library_gen(media_type)(): - prov_item: MediaItemType = prov_item - - db_item: MediaItemType = await controller.get_db_item_by_prov_id( - provider_item_id=prov_item.item_id, - provider=prov_item.provider, - ) - if not db_item: - # dump the item in the db, rich metadata is lazy loaded later - db_item = await controller.add_db_item(prov_item) - elif ( - db_item.metadata.checksum and prov_item.metadata.checksum - ) and db_item.metadata.checksum != prov_item.metadata.checksum: - # item checksum changed - db_item = await controller.update_db_item( - db_item.item_id, prov_item + async with self.mass.database.get_db() as db: + for media_type in self.supported_mediatypes: + self.logger.debug("Start sync of %s items.", media_type.value) + controller = self.mass.music.get_controller(media_type) + + # create a set of all previous and current db id's + prev_ids = set() + for db_item in await controller.library(): + for prov_id in db_item.provider_ids: + if prov_id.prov_id == self.id: + prev_ids.add(db_item.item_id) + cur_ids = set() + async for prov_item in self._get_library_gen(media_type)(): + prov_item: MediaItemType = prov_item + + db_item: MediaItemType = await controller.get_db_item_by_prov_id( + provider_item_id=prov_item.item_id, + provider=prov_item.provider, + db=db, ) - cur_ids.add(db_item.item_id) - if not db_item.in_library: - await controller.set_db_library(db_item.item_id, True) - - # process deletions - for item_id in prev_ids: - if item_id not in cur_ids: - # only mark the item as not in library and leave the metadata in db - await controller.set_db_library(item_id, False) + if not db_item: + # dump the item in the db, rich metadata is lazy loaded later + db_item = await controller.add_db_item(prov_item, db=db) + elif ( + db_item.metadata.checksum and prov_item.metadata.checksum + ) and db_item.metadata.checksum != prov_item.metadata.checksum: + # item checksum changed + db_item = await controller.update_db_item( + db_item.item_id, prov_item, db=db + ) + cur_ids.add(db_item.item_id) + if not db_item.in_library: + await controller.set_db_library(db_item.item_id, True, db=db) + + # process deletions + for item_id in prev_ids: + if item_id not in cur_ids: + # only mark the item as not in library and leave the metadata in db + await controller.set_db_library(item_id, False, db=db) # DO NOT OVERRIDE BELOW