From: Marcel van der Veldt Date: Fri, 13 Nov 2020 17:05:54 +0000 (+0100) Subject: version 0.0.64 X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=20420784c4bfdf520dba5679b0bf305f3b28d6f4;p=music-assistant-server.git version 0.0.64 finished datamodel changes --- diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8c801b21..c2b300b9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: daily + interval: weekly - package-ecosystem: "pip" directory: "/" schedule: diff --git a/music_assistant/constants.py b/music_assistant/constants.py index c52ca59c..790a7d07 100755 --- a/music_assistant/constants.py +++ b/music_assistant/constants.py @@ -1,6 +1,6 @@ """All constants for Music Assistant.""" -__version__ = "0.0.63" +__version__ = "0.0.64" REQUIRED_PYTHON_VER = "3.8" # configuration keys/attributes diff --git a/music_assistant/helpers/compare.py b/music_assistant/helpers/compare.py new file mode 100644 index 00000000..579c76c3 --- /dev/null +++ b/music_assistant/helpers/compare.py @@ -0,0 +1,90 @@ +"""Several helper/utils to compare objects.""" +import re +from typing import List + +import unidecode +from music_assistant.models.media_types import Album, Artist, Track + + +def get_compare_string(input_str): + """Return clean lowered string for compare actions.""" + unaccented_string = unidecode.unidecode(input_str) + return re.sub(r"[^a-zA-Z0-9]", "", unaccented_string).lower() + + +def compare_strings(str1, str2, strict=False): + """Compare strings and return True if we have an (almost) perfect match.""" + match = str1.lower() == str2.lower() + if not match and not strict: + match = get_compare_string(str1) == get_compare_string(str2) + return match + + +def compare_artists(left_artists: List[Artist], right_artists: List[Artist]): + """Compare two lists of artist and return True if a match was found.""" + for left_artist in left_artists: + for right_artist in right_artists: + if compare_strings(left_artist.name, right_artist.name): + return True + return False + + +def compare_albums(left_albums: List[Album], right_albums: List[Album]): + """Compare two lists of albums and return True if a match was found.""" + for left_album in left_albums: + for right_album in right_albums: + if compare_album(left_album, right_album): + return True + return False + + +def compare_album(left_album: Album, right_album: Album): + """Compare two album items and return True if they match.""" + if ( + left_album.provider == right_album.provider + and left_album.item_id == right_album.item_id + ): + return True + if left_album.upc and left_album.upc == right_album.upc: + # UPC is always 100% accurate match + return True + if not compare_strings(left_album.name, right_album.name): + return False + if not compare_strings(left_album.version, right_album.version): + return False + if not compare_strings(left_album.artist.name, right_album.artist.name): + return False + if left_album.year != right_album.year: + return False + # 100% match, all criteria passed + return True + + +def compare_track(left_track: Track, right_track: Track): + """Compare two track items and return True if they match.""" + if ( + left_track.provider == right_track.provider + and left_track.item_id == right_track.item_id + ): + return True + if left_track.isrc and left_track.isrc == right_track.isrc: + # ISRC is always 100% accurate match + return True + # track name and version must match + if not compare_strings(left_track.name, right_track.name): + return False + if not compare_strings(left_track.version, right_track.version): + return False + # track artist(s) must match + if not compare_artists(left_track.artists, right_track.artists): + return False + # album match OR near exact duration match + left_albums = left_track.albums or [left_track.album] + right_albums = right_track.albums or [right_track.album] + if not ( + compare_albums(left_albums, right_albums) + or abs(left_track.duration - right_track.duration) <= 5 + ): + return False + # 100% match, all criteria passed + return True diff --git a/music_assistant/helpers/migration.py b/music_assistant/helpers/migration.py new file mode 100644 index 00000000..430fa08d --- /dev/null +++ b/music_assistant/helpers/migration.py @@ -0,0 +1,184 @@ +"""Logic to handle database/configuration changes and creation.""" + +import os +import shutil + +import aiosqlite +from music_assistant.constants import __version__ as app_version +from music_assistant.helpers.typing import MusicAssistantType +from packaging import version + + +async def check_migrations(mass: MusicAssistantType): + """Check for any migrations that need to be done.""" + + is_fresh_setup = len(mass.config.stored_config.keys()) == 0 + prev_version = version.parse(mass.config.stored_config.get("version", "")) + + # perform version specific migrations + if not is_fresh_setup and prev_version < version.parse("0.0.64"): + await run_migration_0064(mass) + + # store version in config + mass.config.stored_config["version"] = app_version + mass.config.save() + + # create default db tables (if needed) + await async_create_db_tables(mass.database.db_file) + + +async def run_migration_0064(mass: MusicAssistantType): + """Run migration for version 0.0.64.""" + # 0.0.64 introduced major changes to all data models and db structure + # a full refresh of data is unavoidable + data_path = mass.config.data_path + tracks_loudness = [] + + for dbname in ["mass.db", "database.db", "music_assistant.db"]: + filename = os.path.join(data_path, dbname) + if os.path.isfile(filename): + # we try to backup the loudness measurements + async with aiosqlite.connect(filename, timeout=120) as db_conn: + db_conn.row_factory = aiosqlite.Row + sql_query = "SELECT * FROM track_loudness" + for db_row in await db_conn.execute_fetchall(sql_query, ()): + tracks_loudness.append( + ( + db_row["provider_track_id"], + db_row["provider"], + db_row["loudness"], + ) + ) + # remove old db file + os.remove(filename) + + # remove old cache db + for dbname in ["cache.db", ".cache.db"]: + filename = os.path.join(data_path, dbname) + if os.path.isfile(filename): + os.remove(filename) + + # remove old thumbs db + for dirname in ["thumbs", ".thumbs", ".thumbnails"]: + dirname = os.path.join(data_path, dirname) + if os.path.isdir(dirname): + shutil.rmtree(dirname, True) + + # create default db tables (if needed) + await async_create_db_tables(mass.database.db_file) + + # restore loudness measurements + if tracks_loudness: + async with aiosqlite.connect(mass.database.db_file, timeout=120) as db_conn: + sql_query = """INSERT or REPLACE INTO track_loudness + (item_id, provider, loudness) VALUES(?,?,?);""" + for item in tracks_loudness: + await db_conn.execute(sql_query, item) + await db_conn.commit() + + +async def async_create_db_tables(db_file): + """Async initialization.""" + async with aiosqlite.connect(db_file, timeout=120) as db_conn: + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS provider_mappings( + item_id INTEGER NOT NULL, + media_type TEXT NOT NULL, + prov_item_id TEXT NOT NULL, + provider TEXT NOT NULL, + quality INTEGER NOT NULL, + details TEXT NULL, + UNIQUE(item_id, media_type, prov_item_id, provider, quality) + );""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS artists( + item_id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + sort_name TEXT, + musicbrainz_id TEXT NOT NULL UNIQUE, + in_library BOOLEAN DEFAULT 0, + metadata json, + provider_ids json + );""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS albums( + item_id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + sort_name TEXT, + album_type TEXT, + year INTEGER, + version TEXT, + in_library BOOLEAN DEFAULT 0, + upc TEXT, + artist json, + metadata json, + provider_ids json + );""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS tracks( + item_id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + sort_name TEXT, + version TEXT, + duration INTEGER, + in_library BOOLEAN DEFAULT 0, + isrc TEXT, + albums json, + artists json, + metadata json, + provider_ids json + );""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS playlists( + item_id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + sort_name TEXT, + owner TEXT NOT NULL, + is_editable BOOLEAN NOT NULL, + checksum TEXT NOT NULL, + in_library BOOLEAN DEFAULT 0, + metadata json, + provider_ids json, + UNIQUE(name, owner) + );""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS radios( + item_id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + sort_name TEXT, + in_library BOOLEAN DEFAULT 0, + metadata json, + provider_ids json + );""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS track_loudness( + item_id INTEGER NOT NULL, + provider TEXT NOT NULL, + loudness REAL, + UNIQUE(item_id, provider));""" + ) + + await db_conn.execute( + """CREATE TABLE IF NOT EXISTS thumbs( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL, + size INTEGER, + UNIQUE(url, size));""" + ) + + await db_conn.commit() + await db_conn.execute("VACUUM;") + await db_conn.commit() diff --git a/music_assistant/helpers/musicbrainz.py b/music_assistant/helpers/musicbrainz.py index bebcbf84..5795fac4 100644 --- a/music_assistant/helpers/musicbrainz.py +++ b/music_assistant/helpers/musicbrainz.py @@ -8,7 +8,7 @@ from typing import Optional import aiohttp from asyncio_throttle import Throttler from music_assistant.helpers.cache import async_use_cache -from music_assistant.helpers.util import compare_strings, get_compare_string +from music_assistant.helpers.compare import compare_strings, get_compare_string LUCENE_SPECIAL = r'([+\-&|!(){}\[\]\^"~*?:\\\/])' diff --git a/music_assistant/helpers/util.py b/music_assistant/helpers/util.py index b18fa8c7..dd47f739 100755 --- a/music_assistant/helpers/util.py +++ b/music_assistant/helpers/util.py @@ -3,7 +3,6 @@ import asyncio import logging import os import platform -import re import socket import struct import tempfile @@ -13,7 +12,6 @@ from typing import Any, Callable, TypeVar import memory_tempfile import ujson -import unidecode # pylint: disable=invalid-name T = TypeVar("T") @@ -82,15 +80,6 @@ def run_async_background_task(executor, corofn, *args): return asyncio.get_event_loop().run_in_executor(executor, run_task, corofn, *args) -def get_sort_name(name): - """Create a sort name for an artist/title.""" - sort_name = name - for item in ["The ", "De ", "de ", "Les "]: - if name.startswith(item): - sort_name = "".join(name.split(item)[1:]) - return get_compare_string(sort_name) - - def try_parse_int(possible_int): """Try to parse an int.""" try: @@ -234,20 +223,6 @@ def get_folder_size(folderpath): return total_size_gb -def get_compare_string(input_str): - """Return clean lowered string for compare actions.""" - unaccented_string = unidecode.unidecode(input_str) - return re.sub(r"[^a-zA-Z0-9]", "", unaccented_string).lower() - - -def compare_strings(str1, str2, strict=False): - """Compare strings and return True if we have an (almost) perfect match.""" - match = str1.lower() == str2.lower() - if not match and not strict: - match = get_compare_string(str1) == get_compare_string(str2) - return match - - def merge_dict(base_dict: dict, new_dict: dict, allow_overwite=False): """Merge dict without overwriting existing values.""" final_dict = base_dict.copy() @@ -266,11 +241,20 @@ def merge_list(base_list: list, new_list: list): final_list = [] final_list += base_list for item in new_list: + if hasattr(item, "item_id"): + for prov_item in final_list: + if prov_item.item_id == item.item_id: + prov_item = item if item not in final_list: final_list.append(item) return final_list +def unique_item_ids(objects): + """Filter duplicate item id's from list of items.""" + return list({object_.item_id: object_ for object_ in objects}.values()) + + def try_load_json_file(jsonfile): """Try to load json from file.""" try: diff --git a/music_assistant/helpers/web.py b/music_assistant/helpers/web.py index 55ddcf70..77c3145b 100644 --- a/music_assistant/helpers/web.py +++ b/music_assistant/helpers/web.py @@ -8,24 +8,43 @@ from typing import Any import ujson from aiohttp import web +from mashumaro.exceptions import MissingField from music_assistant.helpers.typing import MusicAssistantType -from music_assistant.models.media_types import MediaType +from music_assistant.models.media_types import ( + Album, + Artist, + FullAlbum, + FullTrack, + Playlist, + Radio, + Track, +) async def async_media_items_from_body(mass: MusicAssistantType, data: dict): """Convert posted body data into media items.""" if not isinstance(data, list): data = [data] - media_items = [] - for item in data: - media_item = await mass.music.async_get_item( - item["item_id"], - item["provider"], - MediaType(item["media_type"]), - lazy=True, - ) - media_items.append(media_item) - return media_items + + def media_item_from_dict(media_item): + if media_item["media_type"] == "artist": + return Artist.from_dict(media_item) + if media_item["media_type"] == "album": + try: + return FullAlbum.from_dict(media_item) + except MissingField: + return Album.from_dict(media_item) + if media_item["media_type"] == "track": + try: + return FullTrack.from_dict(media_item) + except MissingField: + return Track.from_dict(media_item) + if media_item["media_type"] == "playlist": + return Playlist.from_dict(media_item) + if media_item["media_type"] == "radio": + return Radio.from_dict(media_item) + + return [media_item_from_dict(x) for x in data] def require_local_subnet(func): @@ -57,7 +76,7 @@ def serialize_values(obj): def get_val(val): if hasattr(val, "to_dict"): return val.to_dict() - if isinstance(val, list): + if isinstance(val, (list, set, filter)): return [get_val(x) for x in val] if isinstance(val, datetime): return val.isoformat() diff --git a/music_assistant/managers/config.py b/music_assistant/managers/config.py index 1b435806..3655974e 100755 --- a/music_assistant/managers/config.py +++ b/music_assistant/managers/config.py @@ -418,7 +418,7 @@ class PlayerSettings(ConfigBaseItem): class ProviderSettings(ConfigBaseItem): - """Configuration class that holds the music provider settings.""" + """Configuration class that holds the provider settings.""" def all_keys(self): """Return all possible keys of this Config object.""" diff --git a/music_assistant/managers/database.py b/music_assistant/managers/database.py index 0ccde1ad..a62d5cc2 100755 --- a/music_assistant/managers/database.py +++ b/music_assistant/managers/database.py @@ -2,210 +2,75 @@ # pylint: disable=too-many-lines import logging import os -from functools import partial -from typing import List +from typing import List, Optional, Union import aiosqlite -from music_assistant.helpers.util import ( - compare_strings, - merge_dict, - merge_list, - try_parse_int, -) +from music_assistant.helpers.compare import compare_album, compare_track +from music_assistant.helpers.util import merge_dict, merge_list, try_parse_int from music_assistant.helpers.web import json_serializer from music_assistant.models.media_types import ( Album, - AlbumArtist, Artist, + FullAlbum, + FullTrack, + ItemMapping, + MediaItem, MediaItemProviderId, MediaType, Playlist, Radio, SearchResult, Track, - TrackAlbum, - TrackArtist, ) LOGGER = logging.getLogger("database") -class DbConnect: - """Helper to initialize the db connection or utilize an existing one.""" - - def __init__(self, dbfile: str, db_conn: aiosqlite.Connection = None): - """Initialize class.""" - self._db_conn_provided = db_conn is not None - self._db_conn = db_conn - self._dbfile = dbfile - - async def __aenter__(self): - """Enter.""" - if not self._db_conn_provided: - self._db_conn = await aiosqlite.connect(self._dbfile, timeout=120) - return self._db_conn - - async def __aexit__(self, exc_type, exc_value, traceback): - """Exit.""" - if not self._db_conn_provided: - await self._db_conn.close() - return False - - class DatabaseManager: """Class that holds the (logic to the) database.""" def __init__(self, mass): """Initialize class.""" self.mass = mass - self._dbfile = os.path.join(mass.config.data_path, "mass.db") - self.db_conn = partial(DbConnect, self._dbfile) - self.cache = {} - - async def async_setup(self): - """Async initialization.""" - async with DbConnect(self._dbfile) as db_conn: - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS provider_mappings( - item_id INTEGER NOT NULL, - media_type TEXT NOT NULL, - prov_item_id TEXT NOT NULL, - provider TEXT NOT NULL, - quality INTEGER NOT NULL, - details TEXT NULL, - UNIQUE(item_id, media_type, prov_item_id, provider, quality) - );""" - ) - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS artists( - item_id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - sort_name TEXT, - musicbrainz_id TEXT NOT NULL UNIQUE, - in_library BOOLEAN DEFAULT 0, - metadata json, - provider_ids json - );""" - ) - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS albums( - item_id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - sort_name TEXT, - album_type TEXT, - year INTEGER, - version TEXT, - in_library BOOLEAN DEFAULT 0, - upc TEXT, - artist json, - metadata json, - provider_ids json, - UNIQUE(item_id, name, version, year) - );""" - ) - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS tracks( - item_id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - sort_name TEXT, - version TEXT, - duration INTEGER, - in_library BOOLEAN DEFAULT 0, - isrc TEXT, - album json, - artists json, - metadata json, - provider_ids json, - UNIQUE(name, version, item_id, duration) - );""" - ) - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS playlists( - item_id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - sort_name TEXT, - owner TEXT NOT NULL, - is_editable BOOLEAN NOT NULL, - checksum TEXT NOT NULL, - in_library BOOLEAN DEFAULT 0, - metadata json, - provider_ids json, - UNIQUE(name, owner) - );""" - ) - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS radios( - item_id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE, - sort_name TEXT, - in_library BOOLEAN DEFAULT 0, - metadata json, - provider_ids json - );""" - ) - - await db_conn.execute( - """CREATE TABLE IF NOT EXISTS track_loudness( - provider_item_id INTEGER NOT NULL, - provider TEXT NOT NULL, - loudness REAL, - UNIQUE(provider_item_id, provider));""" - ) + self._dbfile = os.path.join(mass.config.data_path, "music_assistant.db") - await db_conn.commit() - await db_conn.execute("VACUUM;") - await db_conn.commit() + @property + def db_file(self): + """Return location of database on disk.""" + return self._dbfile async def async_get_item_by_prov_id( self, provider_id: str, prov_item_id: str, media_type: MediaType, - db_conn: aiosqlite.Connection = None, - ) -> int: + ) -> Optional[MediaItem]: """Get the database item for the given prov_id.""" if media_type == MediaType.Artist: - return await self.async_get_artist_by_prov_id( - provider_id, prov_item_id, db_conn - ) + return await self.async_get_artist_by_prov_id(provider_id, prov_item_id) if media_type == MediaType.Album: - return await self.async_get_album_by_prov_id( - provider_id, prov_item_id, db_conn - ) + return await self.async_get_album_by_prov_id(provider_id, prov_item_id) if media_type == MediaType.Track: - return await self.async_get_track_by_prov_id( - provider_id, prov_item_id, db_conn - ) + return await self.async_get_track_by_prov_id(provider_id, prov_item_id) if media_type == MediaType.Playlist: - return await self.async_get_playlist_by_prov_id( - provider_id, prov_item_id, db_conn - ) + return await self.async_get_playlist_by_prov_id(provider_id, prov_item_id) if media_type == MediaType.Radio: - return await self.async_get_radio_by_prov_id( - provider_id, prov_item_id, db_conn - ) + return await self.async_get_radio_by_prov_id(provider_id, prov_item_id) return None async def async_get_track_by_prov_id( self, provider_id: str, prov_item_id: str, - db_conn: aiosqlite.Connection = None, - ) -> int: + ) -> Optional[FullTrack]: """Get the database track for the given prov_id.""" if provider_id == "database": - return await self.async_get_track(prov_item_id, db_conn=db_conn) + return await self.async_get_track(prov_item_id) sql_query = f"""WHERE item_id in (SELECT item_id FROM provider_mappings WHERE prov_item_id = '{prov_item_id}' AND provider = '{provider_id}' AND media_type = 'track')""" - for item in await self.async_get_tracks(sql_query, db_conn=db_conn): + for item in await self.async_get_tracks(sql_query): return item return None @@ -213,16 +78,15 @@ class DatabaseManager: self, provider_id: str, prov_item_id: str, - db_conn: aiosqlite.Connection = None, - ) -> int: + ) -> Optional[FullAlbum]: """Get the database album for the given prov_id.""" if provider_id == "database": - return await self.async_get_album(prov_item_id, db_conn=db_conn) + return await self.async_get_album(prov_item_id) sql_query = f"""WHERE item_id in (SELECT item_id FROM provider_mappings WHERE prov_item_id = '{prov_item_id}' AND provider = '{provider_id}' AND media_type = 'album')""" - for item in await self.async_get_albums(sql_query, db_conn=db_conn): + for item in await self.async_get_albums(sql_query): return item return None @@ -230,33 +94,29 @@ class DatabaseManager: self, provider_id: str, prov_item_id: str, - db_conn: aiosqlite.Connection = None, - ) -> int: + ) -> Optional[Artist]: """Get the database artist for the given prov_id.""" if provider_id == "database": - return await self.async_get_artist(prov_item_id, db_conn=db_conn) + return await self.async_get_artist(prov_item_id) sql_query = f"""WHERE item_id in (SELECT item_id FROM provider_mappings WHERE prov_item_id = '{prov_item_id}' AND provider = '{provider_id}' AND media_type = 'artist')""" - for item in await self.async_get_artists(sql_query, db_conn=db_conn): + for item in await self.async_get_artists(sql_query): return item return None async def async_get_playlist_by_prov_id( - self, - provider_id: str, - prov_item_id: str, - db_conn: aiosqlite.Connection = None, - ) -> int: + self, provider_id: str, prov_item_id: str + ) -> Optional[Playlist]: """Get the database playlist for the given prov_id.""" if provider_id == "database": - return await self.async_get_playlist(prov_item_id, db_conn=db_conn) + return await self.async_get_playlist(prov_item_id) sql_query = f"""WHERE item_id in (SELECT item_id FROM provider_mappings WHERE prov_item_id = '{prov_item_id}' AND provider = '{provider_id}' AND media_type = 'playlist')""" - for item in await self.async_get_playlists(sql_query, db_conn=db_conn): + for item in await self.async_get_playlists(sql_query): return item return None @@ -264,16 +124,15 @@ class DatabaseManager: self, provider_id: str, prov_item_id: str, - db_conn: aiosqlite.Connection = None, - ) -> int: + ) -> Optional[Radio]: """Get the database radio for the given prov_id.""" if provider_id == "database": - return await self.async_get_radio(prov_item_id, db_conn=db_conn) + return await self.async_get_radio(prov_item_id) sql_query = f"""WHERE item_id in (SELECT item_id FROM provider_mappings WHERE prov_item_id = '{prov_item_id}' AND provider = '{provider_id}' AND media_type = 'radio')""" - for item in await self.async_get_radios(sql_query, db_conn=db_conn): + for item in await self.async_get_radios(sql_query): return item return None @@ -281,29 +140,24 @@ class DatabaseManager: self, searchquery: str, media_types: List[MediaType] ) -> SearchResult: """Search library for the given searchphrase.""" - async with DbConnect(self._dbfile) as db_conn: - result = SearchResult([], [], [], [], []) - searchquery = "%" + searchquery + "%" - if media_types is None or MediaType.Artist in media_types: - sql_query = ' WHERE name LIKE "%s"' % searchquery - result.artists = await self.async_get_artists( - sql_query, db_conn=db_conn - ) - if media_types is None or MediaType.Album in media_types: - sql_query = ' WHERE name LIKE "%s"' % searchquery - result.albums = await self.async_get_albums(sql_query, db_conn=db_conn) - if media_types is None or MediaType.Track in media_types: - sql_query = ' WHERE name LIKE "%s"' % searchquery - result.tracks = await self.async_get_tracks(sql_query, db_conn=db_conn) - if media_types is None or MediaType.Playlist in media_types: - sql_query = ' WHERE name LIKE "%s"' % searchquery - result.playlists = await self.async_get_playlists( - sql_query, db_conn=db_conn - ) - if media_types is None or MediaType.Radio in media_types: - sql_query = ' WHERE name LIKE "%s"' % searchquery - result.radios = await self.async_get_radios(sql_query, db_conn=db_conn) - return result + result = SearchResult([], [], [], [], []) + searchquery = "%" + searchquery + "%" + if media_types is None or MediaType.Artist in media_types: + sql_query = ' WHERE name LIKE "%s"' % searchquery + result.artists = await self.async_get_artists(sql_query) + if media_types is None or MediaType.Album in media_types: + sql_query = ' WHERE name LIKE "%s"' % searchquery + result.albums = await self.async_get_albums(sql_query) + if media_types is None or MediaType.Track in media_types: + sql_query = ' WHERE name LIKE "%s"' % searchquery + result.tracks = await self.async_get_tracks(sql_query) + if media_types is None or MediaType.Playlist in media_types: + sql_query = ' WHERE name LIKE "%s"' % searchquery + result.playlists = await self.async_get_playlists(sql_query) + if media_types is None or MediaType.Radio in media_types: + sql_query = ' WHERE name LIKE "%s"' % searchquery + result.radios = await self.async_get_radios(sql_query) + return result async def async_get_library_artists(self, orderby: str = "name") -> List[Artist]: """Get all library artists.""" @@ -338,10 +192,9 @@ class DatabaseManager: self, filter_query: str = None, orderby: str = "name", - db_conn: aiosqlite.Connection = None, ) -> List[Playlist]: """Get all playlists from database.""" - async with DbConnect(self._dbfile, db_conn) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row sql_query = "SELECT * FROM playlists" if filter_query: @@ -352,14 +205,10 @@ class DatabaseManager: for db_row in await db_conn.execute_fetchall(sql_query, ()) ] - async def async_get_playlist( - self, item_id: int, db_conn: aiosqlite.Connection = None - ) -> Playlist: + async def async_get_playlist(self, item_id: int) -> Playlist: """Get playlist record by id.""" item_id = try_parse_int(item_id) - for item in await self.async_get_playlists( - f"WHERE item_id = {item_id}", db_conn=db_conn - ): + for item in await self.async_get_playlists(f"WHERE item_id = {item_id}"): return item return None @@ -374,33 +223,29 @@ class DatabaseManager: if filter_query: sql_query += " " + filter_query sql_query += " ORDER BY %s" % orderby - async with DbConnect(self._dbfile, db_conn) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row return [ Radio.from_db_row(db_row) for db_row in await db_conn.execute_fetchall(sql_query, ()) ] - async def async_get_radio( - self, item_id: int, db_conn: aiosqlite.Connection = None - ) -> Playlist: + async def async_get_radio(self, item_id: int) -> Playlist: """Get radio record by id.""" item_id = try_parse_int(item_id) - for item in await self.async_get_radios( - f"WHERE item_id = {item_id}", db_conn=db_conn - ): + for item in await self.async_get_radios(f"WHERE item_id = {item_id}"): return item return None async def async_add_playlist(self, playlist: Playlist): """Add a new playlist record to the database.""" assert playlist.name - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row cur_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM playlists WHERE name=? AND owner=?;", (playlist.name, playlist.owner), + db_conn, ) if cur_item: @@ -424,12 +269,12 @@ class DatabaseManager: ) as cursor: last_row_id = cursor.lastrowid new_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM playlists WHERE ROWID=?;", (last_row_id,), + db_conn, ) await self.__async_add_prov_ids( - new_item[0], MediaType.Playlist, playlist.provider_ids, db_conn + new_item[0], MediaType.Playlist, playlist.provider_ids, db_conn=db_conn ) await db_conn.commit() LOGGER.debug("added playlist %s to database", playlist.name) @@ -438,11 +283,11 @@ class DatabaseManager: async def async_update_playlist(self, item_id: int, playlist: Playlist): """Update a playlist record in the database.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row cur_item = Playlist.from_db_row( await self.__execute_fetchone( - db_conn, "SELECT * FROM playlists WHERE item_id=?;", (item_id,) + "SELECT * FROM playlists WHERE item_id=?;", (item_id,), db_conn ) ) metadata = merge_dict(cur_item.metadata, playlist.metadata) @@ -470,7 +315,7 @@ class DatabaseManager: ), ) await self.__async_add_prov_ids( - item_id, MediaType.Playlist, playlist.provider_ids, db_conn + item_id, MediaType.Playlist, playlist.provider_ids, db_conn=db_conn ) LOGGER.debug("updated playlist %s in database: %s", playlist.name, item_id) await db_conn.commit() @@ -480,19 +325,17 @@ class DatabaseManager: async def async_add_radio(self, radio: Radio): """Add a new radio record to the database.""" assert radio.name - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row cur_item = await self.__execute_fetchone( - db_conn, - "SELECT (item_id) FROM radios WHERE name=?;", - (radio.name,), + "SELECT (item_id) FROM radios WHERE name=?;", (radio.name,), db_conn ) if cur_item: # update existing return await self.async_update_radio(cur_item[0], radio) # insert radio sql_query = """INSERT INTO radios (name, sort_name, metadata, provider_ids) - VALUES(?,?,?);""" + VALUES(?,?,?,?);""" async with db_conn.execute( sql_query, ( @@ -504,12 +347,12 @@ class DatabaseManager: ) as cursor: last_row_id = cursor.lastrowid new_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM radios WHERE ROWID=?;", (last_row_id,), + db_conn, ) await self.__async_add_prov_ids( - new_item[0], MediaType.Radio, radio.provider_ids, db_conn + new_item[0], MediaType.Radio, radio.provider_ids, db_conn=db_conn ) await db_conn.commit() LOGGER.debug("added radio %s to database", radio.name) @@ -518,11 +361,11 @@ class DatabaseManager: async def async_update_radio(self, item_id: int, radio: Radio): """Update a radio record in the database.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row cur_item = Radio.from_db_row( await self.__execute_fetchone( - db_conn, "SELECT * FROM radios WHERE item_id=?;", (item_id,) + "SELECT * FROM radios WHERE item_id=?;", (item_id,), db_conn ) ) metadata = merge_dict(cur_item.metadata, radio.metadata) @@ -544,7 +387,7 @@ class DatabaseManager: ), ) await self.__async_add_prov_ids( - item_id, MediaType.Radio, radio.provider_ids, db_conn + item_id, MediaType.Radio, radio.provider_ids, db_conn=db_conn ) LOGGER.debug("updated radio %s in database: %s", radio.name, item_id) await db_conn.commit() @@ -555,7 +398,7 @@ class DatabaseManager: self, item_id: int, media_type: MediaType, provider: str ): """Add an item to the library (item must already be present in the db!).""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: item_id = try_parse_int(item_id) db_name = media_type.value + "s" sql_query = f"UPDATE {db_name} SET in_library=1 WHERE item_id=?;" @@ -566,7 +409,7 @@ class DatabaseManager: self, item_id: int, media_type: MediaType, provider: str ): """Remove item from the library.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: item_id = try_parse_int(item_id) db_name = media_type.value + "s" sql_query = f"UPDATE {db_name} SET in_library=0 WHERE item_id=?;" @@ -584,33 +427,28 @@ class DatabaseManager: if filter_query: sql_query += " " + filter_query sql_query += " ORDER BY %s" % orderby - async with DbConnect(self._dbfile, db_conn) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row return [ Artist.from_db_row(db_row) for db_row in await db_conn.execute_fetchall(sql_query, ()) ] - async def async_get_artist( - self, item_id: int, db_conn: aiosqlite.Connection = None - ) -> Artist: + async def async_get_artist(self, item_id: int) -> Artist: """Get artist record by id.""" item_id = try_parse_int(item_id) - for item in await self.async_get_artists( - "WHERE item_id = %d" % item_id, db_conn=db_conn - ): + for item in await self.async_get_artists("WHERE item_id = %d" % item_id): return item return None async def async_add_artist(self, artist: Artist): """Add a new artist record to the database.""" - assert artist.musicbrainz_id - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row cur_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM artists WHERE musicbrainz_id=?;", (artist.musicbrainz_id,), + db_conn, ) if cur_item: # update existing @@ -631,53 +469,49 @@ class DatabaseManager: ) as cursor: last_row_id = cursor.lastrowid new_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM artists WHERE ROWID=?;", (last_row_id,), + db_conn, ) await self.__async_add_prov_ids( - new_item[0], MediaType.Artist, artist.provider_ids, db_conn + new_item[0], MediaType.Artist, artist.provider_ids, db_conn=db_conn ) await db_conn.commit() - LOGGER.debug("added artist %s to database", artist.name) - # return created object - return await self.async_get_artist(new_item[0]) + LOGGER.debug("added artist %s to database", artist.name) + # return created object + return await self.async_get_artist(new_item[0]) async def async_update_artist(self, item_id: int, artist: Artist): """Update a artist record in the database.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row db_row = await self.__execute_fetchone( - db_conn, "SELECT * FROM artists WHERE item_id=?;", (item_id,) + "SELECT * FROM artists WHERE item_id=?;", (item_id,), db_conn ) cur_item = Artist.from_db_row(db_row) metadata = merge_dict(cur_item.metadata, artist.metadata) provider_ids = merge_list(cur_item.provider_ids, artist.provider_ids) sql_query = """UPDATE artists - SET name=?, - sort_name=?, - musicbrainz_id=?, + SET musicbrainz_id=?, metadata=?, provider_ids=? WHERE item_id=?;""" await db_conn.execute( sql_query, ( - artist.name, - artist.sort_name, - artist.musicbrainz_id, + artist.musicbrainz_id or cur_item.musicbrainz_id, json_serializer(metadata), json_serializer(provider_ids), item_id, ), ) await self.__async_add_prov_ids( - item_id, MediaType.Artist, artist.provider_ids, db_conn + item_id, MediaType.Artist, artist.provider_ids, db_conn=db_conn ) LOGGER.debug("updated artist %s in database: %s", artist.name, item_id) await db_conn.commit() - # return updated object - return await self.async_get_artist(item_id) + # return updated object + return await self.async_get_artist(item_id) async def async_get_albums( self, @@ -690,74 +524,57 @@ class DatabaseManager: if filter_query: sql_query += " " + filter_query sql_query += " ORDER BY %s" % orderby - async with DbConnect(self._dbfile, db_conn) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row return [ Album.from_db_row(db_row) for db_row in await db_conn.execute_fetchall(sql_query, ()) ] - async def async_get_album( - self, item_id: int, db_conn: aiosqlite.Connection = None - ) -> Album: + async def async_get_album(self, item_id: int) -> FullAlbum: """Get album record by id.""" item_id = try_parse_int(item_id) # get from db - for item in await self.async_get_albums( - "WHERE item_id = %d" % item_id, db_conn=db_conn - ): - item.artist = await self.async_get_artist(item.artist.item_id) + for item in await self.async_get_albums("WHERE item_id = %d" % item_id): + item.artist = ( + await self.async_get_artist_by_prov_id( + item.artist.provider, item.artist.item_id + ) + or item.artist + ) return item return None async def async_add_album(self, album: Album): """Add a new album record to the database.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row - + cur_item = None # always try to grab existing item by external_id - cur_item = await self.__execute_fetchone( - db_conn, - "SELECT (item_id) FROM albums WHERE upc=?;", - (album.upc,), - ) - # fallback to matching on artist, name and version + if album.upc: + for item in await self.async_get_albums(f"WHERE upc='{album.upc}'"): + cur_item = item + # fallback to matching if not cur_item: - cur_item = await self.__execute_fetchone( - db_conn, - """SELECT item_id FROM albums WHERE - json_extract("artist", '$.item_id') = ? - AND sort_name=? AND version=? AND year=? AND album_type=?""", - ( - album.artist.item_id, - album.sort_name, - album.version, - int(album.year), - album.album_type.value, - ), - ) - # fallback to almost exact match - if not cur_item: - for item in await db_conn.execute_fetchall( - """SELECT * FROM albums WHERE - json_extract("artist", '$.item_id') = ? - AND sort_name = ?""", - (album.artist.item_id, album.sort_name), + sql_query = "SELECT item_id from albums WHERE sort_name LIKE ?" + for db_row in await db_conn.execute_fetchall( + sql_query, (album.sort_name,) ): - if (not album.version and item["year"] == album.year) or ( - album.version and item["version"] == album.version - ): + item = await self.async_get_album(db_row["item_id"]) + if compare_album(item, album): cur_item = item break - if cur_item: # update existing - return await self.async_update_album(cur_item[0], album) + return await self.async_update_album(cur_item.item_id, album) + # insert album - album_artist = AlbumArtist( - item_id=album.artist.item_id, - provider="database", - name=album.artist.name, + assert album.artist + album_artist = ItemMapping.from_item( + await self.async_get_artist_by_prov_id( + album.artist.provider, album.artist.item_id + ) + or album.artist ) sql_query = """INSERT INTO albums (name, sort_name, album_type, year, version, upc, artist, metadata, provider_ids) @@ -778,41 +595,36 @@ class DatabaseManager: ) as cursor: last_row_id = cursor.lastrowid new_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM albums WHERE ROWID=?;", (last_row_id,), + db_conn, ) await self.__async_add_prov_ids( - new_item[0], MediaType.Album, album.provider_ids, db_conn + new_item[0], MediaType.Album, album.provider_ids, db_conn=db_conn ) await db_conn.commit() - LOGGER.debug("added album %s to database", album.name) - # return created object - return await self.async_get_album(new_item[0]) + LOGGER.debug("added album %s to database", album.name) + # return created object + return await self.async_get_album(new_item[0]) async def async_update_album(self, item_id: int, album: Album): """Update a album record in the database.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row - cur_item = Album.from_db_row( - await self.__execute_fetchone( - db_conn, "SELECT * FROM albums WHERE item_id=?;", (item_id,) + cur_item = await self.async_get_album(item_id) + album_artist = ItemMapping.from_item( + await self.async_get_artist_by_prov_id( + cur_item.artist.provider, cur_item.artist.item_id ) - ) - album_artist = AlbumArtist( - item_id=album.artist.item_id, - provider="database", - name=album.artist.name, + or await self.async_get_artist_by_prov_id( + album.artist.provider, album.artist.item_id + ) + or cur_item.artist ) metadata = merge_dict(cur_item.metadata, album.metadata) provider_ids = merge_list(cur_item.provider_ids, album.provider_ids) sql_query = """UPDATE albums - SET name=?, - sort_name=?, - album_type=?, - year=?, - version=?, - upc=?, + SET upc=?, artist=?, metadata=?, provider_ids=? @@ -820,12 +632,7 @@ class DatabaseManager: await db_conn.execute( sql_query, ( - album.name, - album.sort_name, - album.album_type.value, - album.year, - album.version, - album.upc, + album.upc or cur_item.upc, json_serializer(album_artist), json_serializer(metadata), json_serializer(provider_ids), @@ -833,12 +640,12 @@ class DatabaseManager: ), ) await self.__async_add_prov_ids( - item_id, MediaType.Album, album.provider_ids, db_conn + item_id, MediaType.Album, album.provider_ids, db_conn=db_conn ) LOGGER.debug("updated album %s in database: %s", album.name, item_id) await db_conn.commit() - # return updated object - return await self.async_get_album(item_id) + # return updated object + return await self.async_get_album(item_id) async def async_get_tracks( self, @@ -851,7 +658,7 @@ class DatabaseManager: if filter_query: sql_query += " " + filter_query sql_query += " ORDER BY %s" % orderby - async with DbConnect(self._dbfile, db_conn) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row return [ Track.from_db_row(db_row) @@ -859,92 +666,84 @@ class DatabaseManager: ] async def async_get_tracks_from_provider_ids( - self, - provider_id: str, - prov_item_ids: List[str], + self, provider_id: Union[str, List[str]], prov_item_ids: List[str] ) -> dict: """Get track records for the given prov_ids.""" + provider_ids = provider_id if isinstance(provider_id, list) else [provider_id] + prov_id_str = ",".join([f'"{x}"' for x in provider_ids]) prov_item_id_str = ",".join([f'"{x}"' for x in prov_item_ids]) sql_query = f"""WHERE item_id in (SELECT item_id FROM provider_mappings - WHERE provider = '{provider_id}' AND media_type = 'track' + WHERE provider in ({prov_id_str}) AND media_type = 'track' AND prov_item_id in ({prov_item_id_str}) )""" return await self.async_get_tracks(sql_query) - async def async_get_track( - self, item_id: int, db_conn: aiosqlite.Connection = None - ) -> Track: - """Get track record by id.""" + async def async_get_track(self, item_id: int) -> FullTrack: + """Get full track record by id.""" item_id = try_parse_int(item_id) - for item in await self.async_get_tracks( - "WHERE item_id = %d" % item_id, db_conn=db_conn - ): - item.album = await self.async_get_album(item.album.item_id) - artist_ids = [str(x.item_id) for x in item.artists] - query = "WHERE item_id in (%s)" % ",".join(artist_ids) - item.artists = await self.async_get_artists(query) + for item in await self.async_get_tracks("WHERE item_id = %d" % item_id): + # include full album info + item.albums = list( + filter( + None, + [ + await self.async_get_album_by_prov_id( + album.provider, album.item_id + ) + for album in item.albums + ], + ) + ) + item.album = item.albums[0] + # include full artist info + item.artists = [ + await self.async_get_artist_by_prov_id(artist.provider, artist.item_id) + or artist + for artist in item.artists + ] return item return None async def async_add_track(self, track: Track): """Add a new track record to the database.""" - async with DbConnect(self._dbfile) as db_conn: + assert track.album, "Track is missing album" + assert track.artists, "Track is missing artist(s)" + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row - - # always try to grab existing item by external_id - cur_item = await self.__execute_fetchone( - db_conn, - "SELECT (item_id) FROM tracks WHERE isrc=?;", - (track.isrc,), - ) - # fallback to matching on item_id, name and version + cur_item = None + # always try to grab existing item by matching + if track.isrc: + for item in await self.async_get_tracks(f"WHERE isrc='{track.isrc}'"): + cur_item = item + # fallback to matching if not cur_item: - for item in await db_conn.execute_fetchall( - """SELECT * FROM tracks WHERE - json_extract("album", '$.item_id') = ? - AND sort_name=?""", - ( - track.album.item_id, - track.sort_name, - ), + sql_query = "SELECT item_id FROM tracks WHERE sort_name LIKE ?" + for db_row in await db_conn.execute_fetchall( + sql_query, (track.sort_name,) ): - # we perform an additional safety check on the duration or version - if ( - track.version - and compare_strings(item["version"], track.version) - ) or ( - ( - not track.version - and not item["version"] - and abs(item["duration"] - track.duration) < 10 - ) - ): + item = await self.async_get_track(db_row["item_id"]) + if compare_track(item, track): cur_item = item break - if cur_item: # update existing - return await self.async_update_track(cur_item[0], track) - # insert track + return await self.async_update_track(cur_item.item_id, track) + # Item does not yet exist: Insert track sql_query = """INSERT INTO tracks - (name, sort_name, album, artists, duration, version, isrc, metadata, provider_ids) + (name, sort_name, albums, artists, duration, version, isrc, metadata, provider_ids) VALUES(?,?,?,?,?,?,?,?,?);""" - # we store a simplified artist/album object in tracks - artists = [ - TrackArtist(item_id=x.item_id, provider="database", name=x.name) - for x in track.artists - ] - album = TrackAlbum( - item_id=track.album.item_id, provider="database", name=track.album.name - ) + # we store a mapping to artists and albums on the track for easier access/listings + track_artists = await self.__async_get_track_artists(track) + track_albums = await self.__async_get_track_albums(track) + async with db_conn.execute( sql_query, ( track.name, track.sort_name, - json_serializer(album), - json_serializer(artists), + json_serializer(track_albums), + json_serializer(track_artists), track.duration, track.version, track.isrc, @@ -954,93 +753,73 @@ class DatabaseManager: ) as cursor: last_row_id = cursor.lastrowid new_item = await self.__execute_fetchone( - db_conn, "SELECT (item_id) FROM tracks WHERE ROWID=?;", (last_row_id,), + db_conn, ) await self.__async_add_prov_ids( - new_item[0], MediaType.Track, track.provider_ids, db_conn + new_item[0], MediaType.Track, track.provider_ids, db_conn=db_conn ) await db_conn.commit() - LOGGER.debug("added track %s to database", track.name) - # return created object - return await self.async_get_track(new_item[0]) + LOGGER.debug("added track %s to database", track.name) + # return created object + return await self.async_get_track(new_item[0]) async def async_update_track(self, item_id: int, track: Track): """Update a track record in the database.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: db_conn.row_factory = aiosqlite.Row - cur_item = Track.from_db_row( - await self.__execute_fetchone( - db_conn, "SELECT * FROM tracks WHERE item_id=?;", (item_id,) - ) + cur_item = await self.async_get_track(item_id) + + # we store a mapping to artists and albums on the track for easier access/listings + track_artists = await self.__async_get_track_artists( + track, cur_item.artists ) + track_albums = await self.__async_get_track_albums(track, cur_item.albums) + # merge metadata and provider id's metadata = merge_dict(cur_item.metadata, track.metadata) provider_ids = merge_list(cur_item.provider_ids, track.provider_ids) - artists = [ - TrackArtist(item_id=x.item_id, provider="database", name=x.name) - for x in track.artists - ] - album = TrackAlbum( - item_id=track.album.item_id, provider="database", name=track.album.name - ) sql_query = """UPDATE tracks - SET name=?, - sort_name=?, - album=?, - artists=?, - duration=?, - version=?, - isrc=?, + SET isrc=?, metadata=?, - provider_ids=? + provider_ids=?, + artists=?, + albums=? WHERE item_id=?;""" await db_conn.execute( sql_query, ( - track.name, - track.sort_name, - json_serializer(album), - json_serializer(artists), - track.duration, - track.version, - track.isrc, + track.isrc or cur_item.isrc, json_serializer(metadata), json_serializer(provider_ids), + json_serializer(track_artists), + json_serializer(track_albums), item_id, ), ) await self.__async_add_prov_ids( - item_id, MediaType.Track, track.provider_ids, db_conn + item_id, MediaType.Track, track.provider_ids, db_conn=db_conn ) LOGGER.debug("updated track %s in database: %s", track.name, item_id) await db_conn.commit() - # return updated object - return await self.async_get_track(item_id) - - async def async_get_artist_albums( - self, item_id: int, orderby: str = "name" - ) -> List[Album]: - """Get all library albums for the given artist.""" - # TODO: use json query type instead of text search - sql_query = f"WHERE json_extract(\"artist\", '$.item_id') = {item_id}" - return await self.async_get_albums(sql_query, orderby=orderby) + # return updated object + return await self.async_get_track(item_id) async def async_set_track_loudness( - self, provider_item_id: str, provider: str, loudness: int + self, item_id: str, provider: str, loudness: int ): """Set integrated loudness for a track in db.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: sql_query = """INSERT or REPLACE INTO track_loudness - (provider_item_id, provider, loudness) VALUES(?,?,?);""" - await db_conn.execute(sql_query, (provider_item_id, provider, loudness)) + (item_id, provider, loudness) VALUES(?,?,?);""" + await db_conn.execute(sql_query, (item_id, provider, loudness)) await db_conn.commit() async def async_get_track_loudness(self, provider_item_id, provider): """Get integrated loudness for a track in db.""" - async with DbConnect(self._dbfile) as db_conn: + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: sql_query = """SELECT loudness FROM track_loudness WHERE - provider_item_id = ? AND provider = ?""" + item_id = ? AND provider = ?""" async with db_conn.execute( sql_query, (provider_item_id, provider) ) as cursor: @@ -1049,6 +828,29 @@ class DatabaseManager: return result[0] return None + async def async_get_thumbnail_id(self, url, size): + """Get/create id for thumbnail.""" + async with aiosqlite.connect(self._dbfile, timeout=120) as db_conn: + sql_query = """SELECT id FROM thumbs WHERE + url = ? AND size = ?""" + async with db_conn.execute(sql_query, (url, size)) as cursor: + result = await cursor.fetchone() + if result: + return result[0] + # create if it doesnt exist + sql_query = """INSERT INTO thumbs + (url, size) VALUES(?,?);""" + async with db_conn.execute( + sql_query, + (url, size), + ) as cursor: + last_row_id = cursor.lastrowid + new_item = await self.__execute_fetchone( + "SELECT id FROM thumbs WHERE ROWID=?;", (last_row_id,), db_conn + ) + await db_conn.commit() + return new_item[0] + async def __async_add_prov_ids( self, item_id: int, @@ -1075,9 +877,51 @@ class DatabaseManager: ) async def __execute_fetchone( - self, db_conn: aiosqlite.Connection, query: str, query_params: tuple + self, query: str, query_params: tuple, db_conn: aiosqlite.Connection ): """Return first row of given query.""" for item in await db_conn.execute_fetchall(query, query_params): return item return None + + async def __async_get_track_albums( + self, track: Track, cur_albums: Optional[List[ItemMapping]] = None + ) -> List[ItemMapping]: + """Extract all (unique) albums of track as ItemMapping.""" + if not track.albums: + track.albums.append(track.album) + if cur_albums is None: + cur_albums = [] + track_albums = [] + for album in track.albums + cur_albums: + cur_ids = [x.item_id for x in track_albums] + if isinstance(album, ItemMapping): + track_album = await self.async_get_album_by_prov_id( + album.provider_id, album + ) + else: + track_album = await self.async_add_album(album) + if track_album.item_id not in cur_ids: + track_albums.append(ItemMapping.from_item(album)) + return track_albums + + async def __async_get_track_artists( + self, track: Track, cur_artists: Optional[List[ItemMapping]] = None + ) -> List[ItemMapping]: + """Extract all (unique) artists of track as ItemMapping.""" + if cur_artists is None: + cur_artists = [] + track_artists = [] + for item in cur_artists + track.artists: + cur_names = [x.name for x in track_artists] + cur_ids = [x.item_id for x in track_artists] + track_artist = ( + await self.async_get_artist_by_prov_id(item.provider, item.item_id) + or item + ) + if ( + track_artist.name not in cur_names + and track_artist.item_id not in cur_ids + ): + track_artists.append(ItemMapping.from_item(track_artist)) + return track_artists diff --git a/music_assistant/managers/library.py b/music_assistant/managers/library.py new file mode 100755 index 00000000..36dd8823 --- /dev/null +++ b/music_assistant/managers/library.py @@ -0,0 +1,398 @@ +"""LibraryManager: Orchestrates synchronisation of music providers into the library.""" +import asyncio +import functools +import logging +import time +from typing import Any, List + +from music_assistant.constants import EVENT_MUSIC_SYNC_STATUS, EVENT_PROVIDER_REGISTERED +from music_assistant.helpers.util import callback, run_periodic +from music_assistant.models.media_types import ( + Album, + Artist, + MediaItem, + MediaType, + Playlist, + Radio, + Track, +) +from music_assistant.models.provider import ProviderType + +LOGGER = logging.getLogger("music_manager") + + +def sync_task(desc): + """Return decorator to report a sync task.""" + + def wrapper(func): + @functools.wraps(func) + async def async_wrapped(*args): + method_class = args[0] + prov_id = args[1] + # check if this sync task is not already running + for sync_prov_id, sync_desc in method_class.running_sync_jobs: + if sync_prov_id == prov_id and sync_desc == desc: + LOGGER.debug( + "Syncjob %s for provider %s is already running!", desc, prov_id + ) + return + LOGGER.debug("Start syncjob %s for provider %s.", desc, prov_id) + sync_job = (prov_id, desc) + method_class.running_sync_jobs.append(sync_job) + method_class.mass.signal_event( + EVENT_MUSIC_SYNC_STATUS, method_class.running_sync_jobs + ) + await func(*args) + LOGGER.debug("Finished syncing %s for provider %s", desc, prov_id) + method_class.running_sync_jobs.remove(sync_job) + method_class.mass.signal_event( + EVENT_MUSIC_SYNC_STATUS, method_class.running_sync_jobs + ) + + return async_wrapped + + return wrapper + + +class LibraryManager: + """Manage sync of musicproviders to library.""" + + def __init__(self, mass): + """Initialize class.""" + self.running_sync_jobs = [] + self.mass = mass + self.cache = mass.cache + self.mass.add_event_listener(self.mass_event, [EVENT_PROVIDER_REGISTERED]) + + async def async_setup(self): + """Async initialize of module.""" + # schedule sync task + self.mass.add_job(self.__async_music_providers_sync()) + + @callback + def mass_event(self, msg: str, msg_details: Any): + """Handle message on eventbus.""" + if msg == EVENT_PROVIDER_REGISTERED: + # schedule a sync task when a new provider registers + provider = self.mass.get_provider(msg_details) + if provider.type == ProviderType.MUSIC_PROVIDER: + self.mass.add_job(self.async_music_provider_sync(msg_details)) + + ################ GET MediaItems that are added in the library ################ + + async def async_get_library_artists(self, orderby: str = "name") -> List[Artist]: + """Return all library artists, optionally filtered by provider.""" + return await self.mass.database.async_get_library_artists(orderby=orderby) + + async def async_get_library_albums(self, orderby: str = "name") -> List[Album]: + """Return all library albums, optionally filtered by provider.""" + return await self.mass.database.async_get_library_albums(orderby=orderby) + + async def async_get_library_tracks(self, orderby: str = "name") -> List[Track]: + """Return all library tracks, optionally filtered by provider.""" + return await self.mass.database.async_get_library_tracks(orderby=orderby) + + async def async_get_library_playlists( + self, orderby: str = "name" + ) -> List[Playlist]: + """Return all library playlists, optionally filtered by provider.""" + return await self.mass.database.async_get_library_playlists(orderby=orderby) + + async def async_get_library_radios(self, orderby: str = "name") -> List[Playlist]: + """Return all library radios, optionally filtered by provider.""" + return await self.mass.database.async_get_library_radios(orderby=orderby) + + async def async_get_library_playlist_by_name(self, name: str) -> Playlist: + """Get in-library playlist by name.""" + for playlist in await self.mass.music.async_get_library_playlists(): + if playlist.name == name: + return playlist + return None + + async def async_get_radio_by_name(self, name: str) -> Radio: + """Get in-library radio by name.""" + for radio in await self.mass.music.async_get_library_radios(): + if radio.name == name: + return radio + return None + + async def async_library_add(self, media_items: List[MediaItem]): + """Add media item(s) to the library.""" + result = False + for media_item in media_items: + # add to provider's libraries + for prov in media_item.provider_ids: + provider = self.mass.get_provider(prov.provider) + if provider: + result = await provider.async_library_add( + prov.item_id, media_item.media_type + ) + # mark as library item in internal db + if media_item.provider == "database": + await self.mass.database.async_add_to_library( + media_item.item_id, media_item.media_type, media_item.provider + ) + return result + + async def async_library_remove(self, media_items: List[MediaItem]): + """Remove media item(s) from the library.""" + result = False + for media_item in media_items: + # remove from provider's libraries + for prov in media_item.provider_ids: + provider = self.mass.get_provider(prov.provider) + if provider: + result = await provider.async_library_remove( + prov.item_id, media_item.media_type + ) + # mark as library item in internal db + if media_item.provider == "database": + await self.mass.database.async_remove_from_library( + media_item.item_id, media_item.media_type, media_item.provider + ) + return result + + async def async_add_playlist_tracks(self, db_playlist_id: int, tracks: List[Track]): + """Add tracks to playlist - make sure we dont add duplicates.""" + # we can only edit playlists that are in the database (marked as editable) + playlist = await self.mass.music.async_get_playlist(db_playlist_id, "database") + if not playlist or not playlist.is_editable: + return False + # playlist can only have one provider (for now) + playlist_prov = playlist.provider_ids[0] + # grab all existing track ids in the playlist so we can check for duplicates + cur_playlist_track_ids = [] + for item in await self.mass.music.async_get_playlist_tracks( + playlist_prov.item_id, playlist_prov.provider + ): + cur_playlist_track_ids.append(item.item_id) + cur_playlist_track_ids += [i.item_id for i in item.provider_ids] + track_ids_to_add = [] + for track in tracks: + # check for duplicates + already_exists = track.item_id in cur_playlist_track_ids + for track_prov in track.provider_ids: + if track_prov.item_id in cur_playlist_track_ids: + already_exists = True + if already_exists: + continue + # we can only add a track to a provider playlist if track is available on that provider + # this should all be handled in the frontend but these checks are here just to be safe + # a track can contain multiple versions on the same provider + # simply sort by quality and just add the first one (assuming track is still available) + for track_version in sorted( + track.provider_ids, key=lambda x: x.quality, reverse=True + ): + if track_version.provider == playlist_prov.provider: + track_ids_to_add.append(track_version.item_id) + break + if playlist_prov.provider == "file": + # the file provider can handle uri's from all providers so simply add the uri + uri = f"{track_version.provider}://{track_version.item_id}" + track_ids_to_add.append(uri) + break + # actually add the tracks to the playlist on the provider + if track_ids_to_add: + # invalidate cache + await self.mass.database.async_update_playlist( + playlist.item_id, "checksum", str(time.time()) + ) + # return result of the action on the provider + provider = self.mass.get_provider(playlist_prov.provider) + return await provider.async_add_playlist_tracks( + playlist_prov.item_id, track_ids_to_add + ) + return False + + async def async_remove_playlist_tracks(self, db_playlist_id, tracks: List[Track]): + """Remove tracks from playlist.""" + # we can only edit playlists that are in the database (marked as editable) + playlist = await self.mass.music.async_get_playlist(db_playlist_id, "database") + if not playlist or not playlist.is_editable: + return False + # playlist can only have one provider (for now) + prov_playlist = playlist.provider_ids[0] + track_ids_to_remove = [] + for track in tracks: + # a track can contain multiple versions on the same provider, remove all + for track_provider in track.provider_ids: + if track_provider.provider == prov_playlist.provider: + track_ids_to_remove.append(track_provider.item_id) + # actually remove the tracks from the playlist on the provider + if track_ids_to_remove: + # invalidate cache + await self.mass.database.async_update_playlist( + playlist.item_id, "checksum", str(time.time()) + ) + provider = self.mass.get_provider(prov_playlist.provider) + return await provider.async_remove_playlist_tracks( + prov_playlist.item_id, track_ids_to_remove + ) + + @run_periodic(3600 * 3) + async def __async_music_providers_sync(self): + """Periodic sync of all music providers.""" + await asyncio.sleep(10) + for prov in self.mass.get_providers(ProviderType.MUSIC_PROVIDER): + await self.async_music_provider_sync(prov.id) + + async def async_music_provider_sync(self, prov_id: str): + """ + Sync a music provider. + + param prov_id: {string} -- provider id to sync + """ + provider = self.mass.get_provider(prov_id) + if not provider: + return + if MediaType.Album in provider.supported_mediatypes: + await self.async_library_albums_sync(prov_id) + if MediaType.Track in provider.supported_mediatypes: + await self.async_library_tracks_sync(prov_id) + if MediaType.Artist in provider.supported_mediatypes: + await self.async_library_artists_sync(prov_id) + if MediaType.Playlist in provider.supported_mediatypes: + await self.async_library_playlists_sync(prov_id) + if MediaType.Radio in provider.supported_mediatypes: + await self.async_library_radios_sync(prov_id) + + @sync_task("artists") + async def async_library_artists_sync(self, provider_id: str): + """Sync library artists for given provider.""" + music_provider = self.mass.get_provider(provider_id) + cache_key = f"library_artists_{provider_id}" + prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) + cur_db_ids = [] + for item in await music_provider.async_get_library_artists(): + db_item = await self.mass.music.async_get_artist(item.item_id, provider_id) + cur_db_ids.append(db_item.item_id) + await self.mass.database.async_add_to_library( + db_item.item_id, MediaType.Artist, provider_id + ) + # process deletions + for db_id in prev_db_ids: + if db_id not in cur_db_ids: + await self.mass.database.async_remove_from_library( + db_id, MediaType.Artist, provider_id + ) + # store ids in cache for next sync + await self.mass.cache.async_set(cache_key, cur_db_ids) + + @sync_task("albums") + async def async_library_albums_sync(self, provider_id: str): + """Sync library albums for given provider.""" + music_provider = self.mass.get_provider(provider_id) + cache_key = f"library_albums_{provider_id}" + prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) + cur_db_ids = [] + for item in await music_provider.async_get_library_albums(): + db_album = await self.mass.music.async_get_album(item.item_id, provider_id) + if db_album.available != item.available: + # album availability changed, sort this out with auto matching magic + db_album = await self.mass.music.async_match_album(db_album) + cur_db_ids.append(db_album.item_id) + await self.mass.database.async_add_to_library( + db_album.item_id, MediaType.Album, provider_id + ) + # precache album tracks + for album_track in await self.mass.music.async_get_album_tracks( + item.item_id, provider_id + ): + # try to find substitutes for unavailable tracks with matching technique + if not album_track.available: + if album_track.provider == "database": + await self.mass.music.async_match_track(album_track) + else: + await self.mass.music.async_add_track(album_track) + # process album deletions + for db_id in prev_db_ids: + if db_id not in cur_db_ids: + await self.mass.database.async_remove_from_library( + db_id, MediaType.Album, provider_id + ) + # store ids in cache for next sync + await self.mass.cache.async_set(cache_key, cur_db_ids) + + @sync_task("tracks") + async def async_library_tracks_sync(self, provider_id: str): + """Sync library tracks for given provider.""" + music_provider = self.mass.get_provider(provider_id) + cache_key = f"library_tracks_{provider_id}" + prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) + cur_db_ids = [] + for item in await music_provider.async_get_library_tracks(): + db_item = await self.mass.music.async_get_track(item.item_id, provider_id) + if db_item.available != item.available: + # track availability changed, sort this out with auto matching magic + db_item = await self.mass.music.async_add_track(item) + cur_db_ids.append(db_item.item_id) + if db_item.item_id not in prev_db_ids: + await self.mass.database.async_add_to_library( + db_item.item_id, MediaType.Track, provider_id + ) + # process deletions + for db_id in prev_db_ids: + if db_id not in cur_db_ids: + await self.mass.database.async_remove_from_library( + db_id, MediaType.Track, provider_id + ) + # store ids in cache for next sync + await self.mass.cache.async_set(cache_key, cur_db_ids) + + @sync_task("playlists") + async def async_library_playlists_sync(self, provider_id: str): + """Sync library playlists for given provider.""" + music_provider = self.mass.get_provider(provider_id) + cache_key = f"library_playlists_{provider_id}" + prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) + cur_db_ids = [] + for playlist in await music_provider.async_get_library_playlists(): + db_item = await self.mass.music.async_get_playlist( + playlist.item_id, provider_id + ) + if db_item.checksum != playlist.checksum: + db_item = await self.mass.database.async_add_playlist(playlist) + cur_db_ids.append(db_item.item_id) + await self.mass.database.async_add_to_library( + db_item.item_id, MediaType.Playlist, playlist.provider + ) + # precache playlist tracks + for playlist_track in await self.mass.music.async_get_playlist_tracks( + playlist.item_id, provider_id + ): + # try to find substitutes for unavailable tracks with matching technique + if not playlist_track.available: + if playlist_track.provider == "database": + await self.mass.music.async_match_track(playlist_track) + else: + await self.mass.music.async_add_track(playlist_track) + # process playlist deletions + for db_id in prev_db_ids: + if db_id not in cur_db_ids: + await self.mass.database.async_remove_from_library( + db_id, MediaType.Playlist, provider_id + ) + # store ids in cache for next sync + await self.mass.cache.async_set(cache_key, cur_db_ids) + + @sync_task("radios") + async def async_library_radios_sync(self, provider_id: str): + """Sync library radios for given provider.""" + music_provider = self.mass.get_provider(provider_id) + cache_key = f"library_radios_{provider_id}" + prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) + cur_db_ids = [] + for item in await music_provider.async_get_library_radios(): + db_radio = await self.mass.music.async_get_radio(item.item_id, provider_id) + cur_db_ids.append(db_radio.item_id) + await self.mass.database.async_add_to_library( + db_radio.item_id, MediaType.Radio, provider_id + ) + # process deletions + for db_id in prev_db_ids: + if db_id not in cur_db_ids: + await self.mass.database.async_remove_from_library( + db_id, MediaType.Radio, provider_id + ) + # store ids in cache for next sync + await self.mass.cache.async_set(cache_key, cur_db_ids) diff --git a/music_assistant/managers/music.py b/music_assistant/managers/music.py index e749c440..0457165b 100755 --- a/music_assistant/managers/music.py +++ b/music_assistant/managers/music.py @@ -1,22 +1,23 @@ """MusicManager: Orchestrates all data from music providers and sync to internal database.""" -# pylint: disable=too-many-lines + import asyncio -import base64 -import functools import logging -import os -import time -from typing import Any, List, Optional +from typing import List -import aiohttp -from music_assistant.constants import EVENT_MUSIC_SYNC_STATUS, EVENT_PROVIDER_REGISTERED from music_assistant.helpers.cache import async_cached +from music_assistant.helpers.compare import ( + compare_album, + compare_strings, + compare_track, +) from music_assistant.helpers.encryption import async_encrypt_string from music_assistant.helpers.musicbrainz import MusicBrainz -from music_assistant.helpers.util import callback, compare_strings, run_periodic +from music_assistant.helpers.util import unique_item_ids from music_assistant.models.media_types import ( Album, Artist, + FullAlbum, + FullTrack, MediaItem, MediaType, Playlist, @@ -26,87 +27,40 @@ from music_assistant.models.media_types import ( ) from music_assistant.models.provider import MusicProvider, ProviderType from music_assistant.models.streamdetails import ContentType, StreamDetails, StreamType -from PIL import Image LOGGER = logging.getLogger("music_manager") -def sync_task(desc): - """Return decorator to report a sync task.""" - - def wrapper(func): - @functools.wraps(func) - async def async_wrapped(*args): - method_class = args[0] - prov_id = args[1] - # check if this sync task is not already running - for sync_prov_id, sync_desc in method_class.running_sync_jobs: - if sync_prov_id == prov_id and sync_desc == desc: - LOGGER.debug( - "Syncjob %s for provider %s is already running!", desc, prov_id - ) - return - LOGGER.debug("Start syncjob %s for provider %s.", desc, prov_id) - sync_job = (prov_id, desc) - method_class.running_sync_jobs.append(sync_job) - method_class.mass.signal_event( - EVENT_MUSIC_SYNC_STATUS, method_class.running_sync_jobs - ) - await func(*args) - LOGGER.debug("Finished syncing %s for provider %s", desc, prov_id) - method_class.running_sync_jobs.remove(sync_job) - method_class.mass.signal_event( - EVENT_MUSIC_SYNC_STATUS, method_class.running_sync_jobs - ) - - return async_wrapped - - return wrapper - - class MusicManager: """Several helpers around the musicproviders.""" def __init__(self, mass): """Initialize class.""" - self.running_sync_jobs = [] self.mass = mass self.cache = mass.cache self.musicbrainz = MusicBrainz(mass) - self._match_jobs = [] - self.mass.add_event_listener(self.mass_event, [EVENT_PROVIDER_REGISTERED]) async def async_setup(self): """Async initialize of module.""" - # schedule sync task - self.mass.add_job(self.__async_music_providers_sync()) + # nothing to do @property def providers(self) -> List[MusicProvider]: """Return all providers of type musicprovider.""" return self.mass.get_providers(ProviderType.MUSIC_PROVIDER) - @callback - def mass_event(self, msg: str, msg_details: Any): - """Handle message on eventbus.""" - if msg == EVENT_PROVIDER_REGISTERED: - # schedule a sync task when a new provider registers - provider = self.mass.get_provider(msg_details) - if provider.type == ProviderType.MUSIC_PROVIDER: - self.mass.add_job(self.async_music_provider_sync(msg_details)) - ################ GET MediaItem(s) by id and provider ################# async def async_get_item( - self, item_id: str, provider_id: str, media_type: MediaType, lazy: bool = True + self, item_id: str, provider_id: str, media_type: MediaType ): """Get single music item by id and media type.""" if media_type == MediaType.Artist: - return await self.async_get_artist(item_id, provider_id, lazy) + return await self.async_get_artist(item_id, provider_id) if media_type == MediaType.Album: - return await self.async_get_album(item_id, provider_id, lazy) + return await self.async_get_album(item_id, provider_id) if media_type == MediaType.Track: - return await self.async_get_track(item_id, provider_id, lazy) + return await self.async_get_track(item_id, provider_id) if media_type == MediaType.Playlist: return await self.async_get_playlist(item_id, provider_id) if media_type == MediaType.Radio: @@ -114,76 +68,80 @@ class MusicManager: return None async def async_get_artist( - self, item_id: str, provider_id: str, lazy: bool = True + self, item_id: str, provider_id: str, refresh=False ) -> Artist: """Return artist details for the given provider artist id.""" - assert item_id and provider_id + if provider_id == "database" and not refresh: + return await self.mass.database.async_get_artist(item_id) db_item = await self.mass.database.async_get_artist_by_prov_id( provider_id, item_id ) - if not db_item: - # artist not yet in local database so fetch details - provider = self.mass.get_provider(provider_id) - if not provider.available: - return None - cache_key = f"{provider_id}.get_artist.{item_id}" - artist = await async_cached( - self.cache, cache_key, provider.async_get_artist, item_id + if db_item and refresh: + provider_id, item_id = await self.__get_provider_id(db_item) + elif db_item: + return db_item + artist = await self.__async_get_provider_artist(item_id, provider_id) + return await self.async_add_artist(artist) + + async def __async_get_provider_artist( + self, item_id: str, provider_id: str + ) -> Artist: + """Return artist details for the given provider artist id.""" + provider = self.mass.get_provider(provider_id) + if not provider or not provider.available: + raise Exception("Provider %s is not available!" % provider_id) + cache_key = f"{provider_id}.get_artist.{item_id}" + artist = await async_cached( + self.cache, cache_key, provider.async_get_artist, item_id + ) + if not artist: + raise Exception( + "Artist %s not found on provider %s" % (item_id, provider_id) ) - if not artist: - raise Exception( - "Artist %s not found on provider %s" % (item_id, provider_id) - ) - if lazy: - self.mass.add_job(self.__async_add_artist(artist)) - artist.is_lazy = True - return artist - db_item = await self.__async_add_artist(artist) - return db_item + return artist async def async_get_album( - self, - item_id: str, - provider_id: str, - lazy=True, - album_details: Optional[Album] = None, + self, item_id: str, provider_id: str, refresh=False ) -> Album: """Return album details for the given provider album id.""" - assert item_id and provider_id + if provider_id == "database" and not refresh: + return await self.mass.database.async_get_album(item_id) db_item = await self.mass.database.async_get_album_by_prov_id( provider_id, item_id ) - if not db_item: - # album not yet in local database so fetch details - if not album_details: - provider = self.mass.get_provider(provider_id) - if not provider.available: - return None - cache_key = f"{provider_id}.get_album.{item_id}" - album_details = await async_cached( - self.cache, cache_key, provider.async_get_album, item_id - ) - if not album_details: - raise Exception( - "Album %s not found on provider %s" % (item_id, provider_id) - ) - if lazy: - self.mass.add_job(self.__async_add_album(album_details)) - album_details.is_lazy = True - return album_details - db_item = await self.__async_add_album(album_details) - return db_item + if db_item and refresh: + provider_id, item_id = await self.__get_provider_id(db_item) + elif db_item: + return db_item + album = await self.__async_get_provider_album(item_id, provider_id) + return await self.async_add_album(album) + + async def __async_get_provider_album(self, item_id: str, provider_id: str) -> Album: + """Return album details for the given provider album id.""" + provider = self.mass.get_provider(provider_id) + if not provider or not provider.available: + raise Exception("Provider %s is not available!" % provider_id) + cache_key = f"{provider_id}.get_album.{item_id}" + album = await async_cached( + self.cache, cache_key, provider.async_get_album, item_id + ) + if not album: + raise Exception( + "Album %s not found on provider %s" % (item_id, provider_id) + ) + return album async def async_get_track( self, item_id: str, provider_id: str, - lazy: bool = True, track_details: Track = None, + album_details: Album = None, refresh: bool = False, ) -> Track: """Return track details for the given provider track id.""" - assert item_id and provider_id + if provider_id == "database" and not refresh: + return await self.mass.database.async_get_track(item_id) db_item = await self.mass.database.async_get_track_by_prov_id( provider_id, item_id ) @@ -192,30 +150,29 @@ class MusicManager: # it's useful to have the track refreshed from the provider instead of # the database cache to make sure that the track is available and perhaps # another or a higher quality version is available. - if lazy: - self.mass.add_job(self.__async_match_track(db_item)) - else: - await self.__async_match_track(db_item) - if not db_item: - # track not yet in local database so fetch details - if not track_details: - provider = self.mass.get_provider(provider_id) - if not provider.available: - return None - cache_key = f"{provider_id}.get_track.{item_id}" - track_details = await async_cached( - self.cache, cache_key, provider.async_get_track, item_id - ) - if not track_details: - raise Exception( - "Track %s not found on provider %s" % (item_id, provider_id) - ) - if lazy: - self.mass.add_job(self.__async_add_track(track_details)) - track_details.is_lazy = True - return track_details - db_item = await self.__async_add_track(track_details) - return db_item + provider_id, item_id = await self.__get_provider_id(db_item) + elif db_item: + return db_item + if not track_details: + track_details = await self.__async_get_provider_track(item_id, provider_id) + if album_details: + track_details.album = album_details + return await self.async_add_track(track_details) + + async def __async_get_provider_track(self, item_id: str, provider_id: str) -> Album: + """Return track details for the given provider track id.""" + provider = self.mass.get_provider(provider_id) + if not provider or not provider.available: + raise Exception("Provider %s is not available!" % provider_id) + cache_key = f"{provider_id}.get_track.{item_id}" + track = await async_cached( + self.cache, cache_key, provider.async_get_track, item_id + ) + if not track: + raise Exception( + "Track %s not found on provider %s" % (item_id, provider_id) + ) + return track async def async_get_playlist(self, item_id: str, provider_id: str) -> Playlist: """Return playlist details for the given provider playlist id.""" @@ -259,27 +216,25 @@ class MusicManager: item_id = album.provider_ids[0].item_id provider = self.mass.get_provider(provider_id) cache_key = f"{provider_id}.album_tracks.{item_id}" - result = [] - async with self.mass.database.db_conn() as db_conn: - for item in await async_cached( - self.cache, cache_key, provider.async_get_album_tracks, item_id - ): - if not item: - continue - db_item = await self.mass.database.async_get_track_by_prov_id( - item.provider, item.item_id, db_conn - ) - if db_item: - # return database track instead if we have a match - track = db_item - track.disc_number = item.disc_number - track.track_number = item.track_number - else: - track = item - if not track.album: - track.album = album - result.append(track) - return result + all_prov_tracks = await async_cached( + self.cache, cache_key, provider.async_get_album_tracks, item_id + ) + # retrieve list of db items + db_tracks = await self.mass.database.async_get_tracks_from_provider_ids( + [x.provider for x in album.provider_ids], + [x.item_id for x in all_prov_tracks], + ) + # combine provider tracks with db tracks + return [ + await self.__process_item( + item, + db_tracks, + album=album, + disc_number=item.disc_number, + track_number=item.track_number, + ) + for item in all_prov_tracks + ] async def async_get_album_versions( self, item_id: str, provider_id: str @@ -352,472 +307,119 @@ class MusicManager: ) # combine provider tracks with db tracks return [ - await self.__process_track_details(item, index, db_tracks) + await self.__process_item(item, db_tracks, index) for index, item in enumerate(playlist_tracks) ] - async def __process_track_details(self, item, position, db_tracks): - for db_track in db_tracks: - if item.item_id in [x.item_id for x in db_track.provider_ids]: - db_track.position = position - return db_track - item.position = position + async def __process_item( + self, + item, + db_items, + index=None, + album=None, + disc_number=None, + track_number=None, + ): + """Return combined result of provider item and db result.""" + for db_item in db_items: + if item.item_id in [x.item_id for x in db_item.provider_ids]: + item = db_item + break + if index is not None and not item.position: + item.position = index + if album is not None: + item.album = album + if disc_number is not None: + item.disc_number = disc_number + if track_number is not None: + item.track_number = track_number + # make sure artists are unique + if hasattr(item, "artists"): + item.artists = unique_item_ids(item.artists) return item async def async_get_artist_toptracks( self, artist_id: str, provider_id: str ) -> List[Track]: """Return top tracks for an artist.""" - async with self.mass.database.db_conn() as db_conn: - if provider_id == "database": - # tracks from all providers - item_ids = [] - result = [] - artist = await self.mass.database.async_get_artist( - artist_id, db_conn=db_conn - ) - for prov_id in artist.provider_ids: - provider = self.mass.get_provider(prov_id.provider) - if ( - not provider - or MediaType.Track not in provider.supported_mediatypes - ): - continue - for item in await self.async_get_artist_toptracks( - prov_id.item_id, prov_id.provider - ): - if item.item_id not in item_ids: - result.append(item) - item_ids.append(item.item_id) - return result - else: - # items from provider - provider = self.mass.get_provider(provider_id) - cache_key = f"{provider_id}.artist_toptracks.{artist_id}" - result = [] - for item in await async_cached( - self.cache, - cache_key, - provider.async_get_artist_toptracks, - artist_id, - ): - if item: - assert item.item_id and item.provider - db_item = await self.mass.database.async_get_track_by_prov_id( - item.provider, - item.item_id, - db_conn=db_conn, - ) - if db_item: - # return database track instead if we have a match - result.append(db_item) - else: - result.append(item) - return result + artist = await self.async_get_artist(artist_id, provider_id) + # get results from all providers + all_prov_tracks = [ + track + for prov_tracks in await asyncio.gather( + *[ + self.__async_get_provider_artist_toptracks( + item.item_id, item.provider + ) + for item in artist.provider_ids + ] + ) + for track in prov_tracks + ] + # retrieve list of db items + db_tracks = await self.mass.database.async_get_tracks_from_provider_ids( + [x.provider for x in artist.provider_ids], + [x.item_id for x in all_prov_tracks], + ) + # combine provider tracks with db tracks and filter duplicate itemid's + return unique_item_ids( + [await self.__process_item(item, db_tracks) for item in all_prov_tracks] + ) + + async def __async_get_provider_artist_toptracks( + self, artist_id: str, provider_id: str + ) -> List[Track]: + """Return top tracks for an artist on given provider.""" + provider = self.mass.get_provider(provider_id) + if not provider or not provider.available: + LOGGER.error("Provider %s is not available", provider_id) + return [] + cache_key = f"{provider_id}.artist_toptracks.{artist_id}" + return await async_cached( + self.cache, + cache_key, + provider.async_get_artist_toptracks, + artist_id, + ) async def async_get_artist_albums( self, artist_id: str, provider_id: str ) -> List[Album]: """Return (all) albums for an artist.""" - async with self.mass.database.db_conn() as db_conn: - if provider_id == "database": - # albums from all providers - item_ids = [] - result = [] - artist = await self.mass.database.async_get_artist( - artist_id, db_conn=db_conn - ) - for prov_id in artist.provider_ids: - provider = self.mass.get_provider(prov_id.provider) - if ( - not provider - or MediaType.Album not in provider.supported_mediatypes - ): - continue - for item in await self.async_get_artist_albums( - prov_id.item_id, prov_id.provider - ): - if item.item_id not in item_ids: - result.append(item) - item_ids.append(item.item_id) - return result - else: - # items from provider - provider = self.mass.get_provider(provider_id) - cache_key = f"{provider_id}.artist_albums.{artist_id}" - result = [] - for item in await async_cached( - self.cache, cache_key, provider.async_get_artist_albums, artist_id + if provider_id == "database": + # albums from all providers + item_ids = [] + result = [] + artist = await self.mass.database.async_get_artist(artist_id) + for prov_id in artist.provider_ids: + provider = self.mass.get_provider(prov_id.provider) + if not provider or MediaType.Album not in provider.supported_mediatypes: + continue + for item in await self.async_get_artist_albums( + prov_id.item_id, prov_id.provider ): - assert item.item_id and item.provider - db_item = await self.mass.database.async_get_album_by_prov_id( - item.provider, item.item_id, db_conn=db_conn - ) - if db_item: - # return database album instead if we have a match - result.append(db_item) - else: + if item.item_id not in item_ids: result.append(item) - return result - - ################ GET MediaItems that are added in the library ################ - - async def async_get_library_artists(self, orderby: str = "name") -> List[Artist]: - """Return all library artists, optionally filtered by provider.""" - return await self.mass.database.async_get_library_artists(orderby=orderby) - - async def async_get_library_albums(self, orderby: str = "name") -> List[Album]: - """Return all library albums, optionally filtered by provider.""" - return await self.mass.database.async_get_library_albums(orderby=orderby) - - async def async_get_library_tracks(self, orderby: str = "name") -> List[Track]: - """Return all library tracks, optionally filtered by provider.""" - return await self.mass.database.async_get_library_tracks(orderby=orderby) - - async def async_get_library_playlists( - self, orderby: str = "name" - ) -> List[Playlist]: - """Return all library playlists, optionally filtered by provider.""" - return await self.mass.database.async_get_library_playlists(orderby=orderby) - - async def async_get_library_radios(self, orderby: str = "name") -> List[Playlist]: - """Return all library radios, optionally filtered by provider.""" - return await self.mass.database.async_get_library_radios(orderby=orderby) - - ################ ADD MediaItem(s) to database helpers ################ - - async def __async_add_artist(self, artist: Artist) -> int: - """Add artist to local db and return the new database id.""" - if not artist.musicbrainz_id: - artist.musicbrainz_id = await self.__async_get_artist_musicbrainz_id(artist) - # grab additional metadata - artist.metadata = await self.mass.metadata.async_get_artist_metadata( - artist.musicbrainz_id, artist.metadata - ) - db_item = await self.mass.database.async_add_artist(artist) - # also fetch same artist on all providers - await self.__async_match_artist(db_item) - return db_item - - async def __async_add_album(self, album: Album) -> int: - """Add album to local db and return the new database id.""" - # we need to fetch album artist too - album.artist = await self.async_get_artist( - album.artist.item_id, album.artist.provider, lazy=False - ) - db_item = await self.mass.database.async_add_album(album) - # also fetch same album on all providers - await self.__async_match_album(db_item) - return db_item - - async def __async_add_track( - self, track: Track, album_id: Optional[int] = None - ) -> int: - """Add track to local db and return the new database id.""" - track_artists = [] - # we need to fetch track artists too - for track_artist in track.artists: - db_track_artist = await self.async_get_artist( - track_artist.item_id, track_artist.provider, lazy=False - ) - if db_track_artist: - track_artists.append(db_track_artist) - track.artists = track_artists - # fetch album details - prefer optional provided album_id - if album_id: - album_details = await self.async_get_album( - album_id, track.provider, lazy=False - ) - if album_details: - track.album = album_details - # make sure we have a database album - assert track.album - if track.album.provider != "database": - track.album = await self.async_get_album( - track.album.item_id, track.provider, lazy=False - ) - db_item = await self.mass.database.async_add_track(track) - # also fetch same track on all providers (will also get other quality versions) - await self.__async_match_track(db_item) - return db_item - - async def __async_get_artist_musicbrainz_id(self, artist: Artist): - """Fetch musicbrainz id by performing search using the artist name, albums and tracks.""" - # try with album first - for lookup_album in await self.async_get_artist_albums( - artist.item_id, artist.provider - ): - if not lookup_album: - continue - musicbrainz_id = await self.musicbrainz.async_get_mb_artist_id( - artist.name, - albumname=lookup_album.name, - album_upc=lookup_album.upc, - ) - if musicbrainz_id: - return musicbrainz_id - # fallback to track - for lookup_track in await self.async_get_artist_toptracks( - artist.item_id, artist.provider - ): - if not lookup_track: - continue - musicbrainz_id = await self.musicbrainz.async_get_mb_artist_id( - artist.name, - trackname=lookup_track.name, - track_isrc=lookup_track.isrc, - ) - if musicbrainz_id: - return musicbrainz_id - # lookup failed, use the shitty workaround to use the name as id. - LOGGER.warning("Unable to get musicbrainz ID for artist %s !", artist.name) - return artist.name - - async def __async_match_artist(self, artist: Artist): - """ - Try to find matching artists on all providers for the provided (database) artist_id. - - This is used to link objects of different providers together. - :attrib db_artist_id: Database artist_id. - """ - match_job_id = f"artist.{artist.item_id}" - if match_job_id in self._match_jobs: - return - self._match_jobs.append(match_job_id) - cur_providers = [item.provider for item in artist.provider_ids] - for provider in self.mass.get_providers(ProviderType.MUSIC_PROVIDER): - if provider.id in cur_providers: - continue - LOGGER.debug( - "Trying to match artist %s on provider %s", artist.name, provider.name - ) - match_found = False - # try to get a match with some reference albums of this artist - for ref_album in await self.async_get_artist_albums( - artist.item_id, artist.provider + item_ids.append(item.item_id) + return result + else: + # items from provider + provider = self.mass.get_provider(provider_id) + cache_key = f"{provider_id}.artist_albums.{artist_id}" + result = [] + for item in await async_cached( + self.cache, cache_key, provider.async_get_artist_albums, artist_id ): - if match_found: - break - searchstr = "%s - %s" % (artist.name, ref_album.name) - search_result = await self.async_search_provider( - searchstr, provider.id, [MediaType.Album], limit=5 - ) - for strictness in [True, False]: - if match_found: - break - for search_result_item in search_result.albums: - if not search_result_item: - continue - if not compare_strings( - search_result_item.name, ref_album.name, strict=strictness - ): - continue - # double safety check - artist must match exactly ! - if not compare_strings( - search_result_item.artist.name, - artist.name, - strict=strictness, - ): - continue - # just load this item in the database where it will be strictly matched - await self.async_get_artist( - search_result_item.artist.item_id, - search_result_item.artist.provider, - lazy=False, - ) - match_found = True - break - # try to get a match with some reference tracks of this artist - if not match_found: - for search_track in await self.async_get_artist_toptracks( - artist.item_id, artist.provider - ): - if match_found: - break - searchstr = "%s - %s" % (artist.name, search_track.name) - search_results = await self.async_search_provider( - searchstr, provider.id, [MediaType.Track], limit=5 - ) - for strictness in [True, False]: - if match_found: - break - for search_result_item in search_results.tracks: - if match_found: - break - if not search_result_item: - continue - if not compare_strings( - search_result_item.name, - search_track.name, - strict=strictness, - ): - continue - # double safety check - artist must match exactly ! - for match_artist in search_result_item.artists: - if not compare_strings( - match_artist.name, artist.name, strict=strictness - ): - continue - # load this item in the database where it will be strictly matched - await self.async_get_artist( - match_artist.item_id, - match_artist.provider, - lazy=False, - ) - match_found = True - break - if match_found: - LOGGER.debug( - "Found match for Artist %s on provider %s", - artist.name, - provider.name, - ) - else: - LOGGER.warning( - "Could not find match for Artist %s on provider %s", - artist.name, - provider.name, + assert item.item_id and item.provider and item.artist + db_item = await self.mass.database.async_get_album_by_prov_id( + item.provider, item.item_id ) - - async def __async_match_album(self, album: Album): - """ - Try to find matching album on all providers for the provided (database) album_id. - - This is used to link objects of different providers/qualities together. - :attrib db_album_id: Database album_id. - """ - match_job_id = f"album.{album.item_id}" - if match_job_id in self._match_jobs: - return - self._match_jobs.append(match_job_id) - cur_providers = [item.provider for item in album.provider_ids] - providers = self.mass.get_providers(ProviderType.MUSIC_PROVIDER) - for provider in providers: - if provider.id in cur_providers: - continue - LOGGER.debug( - "Trying to match album %s on provider %s", album.name, provider.name - ) - match_found = False - searchstr = "%s - %s" % (album.artist.name, album.name) - if album.version: - searchstr += " " + album.version - search_result = await self.async_search_provider( - searchstr, provider.id, [MediaType.Album], limit=5 - ) - for search_result_item in search_result.albums: - if not search_result_item: - continue - if search_result_item.album_type != album.album_type: - continue - if not ( - compare_strings(search_result_item.name, album.name) - and compare_strings(search_result_item.version, album.version) - ): - continue - if not compare_strings( - search_result_item.artist.name, album.artist.name, strict=False - ): - continue - # just load this item in the database where it will be strictly matched - await self.async_get_album( - search_result_item.item_id, - provider.id, - lazy=False, - album_details=search_result_item, - ) - match_found = True - if match_found: - LOGGER.debug( - "Found match for Album %s on provider %s", album.name, provider.name - ) - else: - LOGGER.warning( - "Could not find match for Album %s on provider %s", - album.name, - provider.name, - ) - - async def __async_match_track(self, track: Track): - """ - Try to find matching track on all providers for the provided (database) track_id. - - This is used to link objects of different providers/qualities together. - :attrib db_track_id: Database track_id. - """ - match_job_id = f"track.{track.item_id}" - if match_job_id in self._match_jobs: - return - self._match_jobs.append(match_job_id) - for provider in self.mass.get_providers(ProviderType.MUSIC_PROVIDER): - LOGGER.debug( - "Trying to match track %s on provider %s", track.name, provider.name - ) - match_found = False - searchstr = "%s - %s" % (track.artists[0].name, track.name) - if track.version: - searchstr += " " + track.version - search_result = await self.async_search_provider( - searchstr, provider.id, [MediaType.Track], limit=10 - ) - for search_result_item in search_result.tracks: - if ( - not search_result_item - or not search_result_item.name - or not search_result_item.album - ): - continue - if not ( - compare_strings(search_result_item.name, track.name) - and compare_strings(search_result_item.version, track.version) - ): - continue - # double safety check - artist must match exactly ! - artist_match_found = False - for artist in track.artists: - if artist_match_found: - break - for search_item_artist in search_result_item.artists: - if not compare_strings( - artist.name, search_item_artist.name, strict=False - ): - continue - # just load this item in the database where it will be strictly matched - await self.async_get_track( - search_item_artist.item_id, - provider.id, - lazy=False, - track_details=search_result_item, - ) - match_found = True - artist_match_found = True - break - if match_found: - LOGGER.debug( - "Found match for Track %s on provider %s", track.name, provider.name - ) - else: - LOGGER.warning( - "Could not find match for Track %s on provider %s", - track.name, - provider.name, - ) - - ################ Various convenience/helper methods ################ - - async def async_get_library_playlist_by_name(self, name: str) -> Playlist: - """Get in-library playlist by name.""" - for playlist in await self.async_get_library_playlists(): - if playlist.name == name: - return playlist - return None - - async def async_get_radio_by_name(self, name: str) -> Radio: - """Get in-library radio by name.""" - for radio in await self.async_get_library_radios(): - if radio.name == name: - return radio - return None + if db_item: + # return database album instead if we have a match + result.append(db_item) + else: + result.append(item) + return result async def async_search_provider( self, @@ -875,192 +477,6 @@ class MusicManager: # TODO: sort by name and filter out duplicates ? return result - async def async_library_add(self, media_items: List[MediaItem]): - """Add media item(s) to the library.""" - result = False - for media_item in media_items: - # make sure we have a database item - db_item = await self.async_get_item( - media_item.item_id, - media_item.provider, - media_item.media_type, - lazy=False, - ) - if not db_item: - continue - # add to provider's libraries - for prov in db_item.provider_ids: - provider = self.mass.get_provider(prov.provider) - if provider: - result = await provider.async_library_add( - prov.item_id, media_item.media_type - ) - # mark as library item in internal db - await self.mass.database.async_add_to_library( - db_item.item_id, db_item.media_type, prov.provider - ) - return result - - async def async_library_remove(self, media_items: List[MediaItem]): - """Remove media item(s) from the library.""" - result = False - for media_item in media_items: - # make sure we have a database item - db_item = await self.async_get_item( - media_item.item_id, - media_item.provider, - media_item.media_type, - lazy=False, - ) - if not db_item: - continue - # remove from provider's libraries - for prov in db_item.provider_ids: - provider = self.mass.get_provider(prov.provider) - if provider: - result = await provider.async_library_remove( - prov.item_id, media_item.media_type - ) - # mark as library item in internal db - await self.mass.database.async_remove_from_library( - db_item.item_id, db_item.media_type, prov.provider - ) - return result - - async def async_add_playlist_tracks(self, db_playlist_id: int, tracks: List[Track]): - """Add tracks to playlist - make sure we dont add duplicates.""" - # we can only edit playlists that are in the database (marked as editable) - playlist = await self.async_get_playlist(db_playlist_id, "database") - if not playlist or not playlist.is_editable: - return False - # playlist can only have one provider (for now) - playlist_prov = playlist.provider_ids[0] - # grab all existing track ids in the playlist so we can check for duplicates - cur_playlist_track_ids = [] - for item in await self.async_get_playlist_tracks( - playlist_prov.item_id, playlist_prov.provider - ): - cur_playlist_track_ids.append(item.item_id) - cur_playlist_track_ids += [i.item_id for i in item.provider_ids] - track_ids_to_add = [] - for track in tracks: - # check for duplicates - already_exists = track.item_id in cur_playlist_track_ids - for track_prov in track.provider_ids: - if track_prov.item_id in cur_playlist_track_ids: - already_exists = True - if already_exists: - continue - # we can only add a track to a provider playlist if track is available on that provider - # this should all be handled in the frontend but these checks are here just to be safe - # a track can contain multiple versions on the same provider - # simply sort by quality and just add the first one (assuming track is still available) - for track_version in sorted( - track.provider_ids, key=lambda x: x.quality, reverse=True - ): - if track_version.provider == playlist_prov.provider: - track_ids_to_add.append(track_version.item_id) - break - if playlist_prov.provider == "file": - # the file provider can handle uri's from all providers so simply add the uri - uri = f"{track_version.provider}://{track_version.item_id}" - track_ids_to_add.append(uri) - break - # actually add the tracks to the playlist on the provider - if track_ids_to_add: - # invalidate cache - await self.mass.database.async_update_playlist( - playlist.item_id, "checksum", str(time.time()) - ) - # return result of the action on the provider - provider = self.mass.get_provider(playlist_prov.provider) - return await provider.async_add_playlist_tracks( - playlist_prov.item_id, track_ids_to_add - ) - return False - - async def async_remove_playlist_tracks(self, db_playlist_id, tracks: List[Track]): - """Remove tracks from playlist.""" - # we can only edit playlists that are in the database (marked as editable) - playlist = await self.async_get_playlist(db_playlist_id, "database") - if not playlist or not playlist.is_editable: - return False - # playlist can only have one provider (for now) - prov_playlist = playlist.provider_ids[0] - track_ids_to_remove = [] - for track in tracks: - # a track can contain multiple versions on the same provider, remove all - for track_provider in track.provider_ids: - if track_provider.provider == prov_playlist.provider: - track_ids_to_remove.append(track_provider.item_id) - # actually remove the tracks from the playlist on the provider - if track_ids_to_remove: - # invalidate cache - await self.mass.database.async_update_playlist( - playlist.item_id, "checksum", str(time.time()) - ) - provider = self.mass.get_provider(prov_playlist.provider) - return await provider.async_remove_playlist_tracks( - prov_playlist.item_id, track_ids_to_remove - ) - - async def async_get_image_thumb( - self, item_id: str, provider_id: str, media_type: MediaType, size: int = 50 - ): - """Get path to (resized) thumb image for given media item.""" - assert item_id and provider_id and media_type - cache_folder = os.path.join(self.mass.config.data_path, ".thumbs") - cache_id = f"{item_id}{media_type}{provider_id}" - cache_id = base64.b64encode(cache_id.encode("utf-8")).decode("utf-8") - cache_file_org = os.path.join(cache_folder, f"{cache_id}0.png") - cache_file_sized = os.path.join(cache_folder, f"{cache_id}{size}.png") - if os.path.isfile(cache_file_sized): - # return file from cache - return cache_file_sized - # no file in cache so we should get it - img_url = "" - # we only retrieve items that we already have in database - item = await self.mass.database.async_get_item_by_prov_id( - provider_id, item_id, media_type - ) - if not item: - return "" - if item and item.metadata.get("image"): - img_url = item.metadata["image"] - elif media_type == MediaType.Track and item.album: - # try album image instead for tracks - return await self.async_get_image_thumb( - item.album.item_id, item.album.provider, MediaType.Album, size - ) - elif media_type == MediaType.Album and item.artist: - # try artist image instead for albums - return await self.async_get_image_thumb( - item.artist.item_id, item.artist.provider, MediaType.Artist, size - ) - if not img_url: - return None - # fetch image and store in cache - os.makedirs(cache_folder, exist_ok=True) - # download base image - async with aiohttp.ClientSession() as session: - async with session.get(img_url, verify_ssl=False) as response: - assert response.status == 200 - img_data = await response.read() - with open(cache_file_org, "wb") as img_file: - img_file.write(img_data) - if not size: - # return base image - return cache_file_org - # save resized image - basewidth = size - img = Image.open(cache_file_org) - wpercent = basewidth / float(img.size[0]) - hsize = int((float(img.size[1]) * float(wpercent))) - img = img.resize((basewidth, hsize), Image.ANTIALIAS) - img.save(cache_file_sized) - # return file from cache - return cache_file_sized - async def async_get_stream_details( self, media_item: MediaItem, player_id: str = "" ) -> StreamDetails: @@ -1090,12 +506,14 @@ class MusicManager: full_track = media_item else: full_track = await self.async_get_track( - media_item.item_id, media_item.provider, lazy=True, refresh=False + media_item.item_id, media_item.provider, refresh=True ) # sort by quality and check track availability for prov_media in sorted( full_track.provider_ids, key=lambda x: x.quality, reverse=True ): + if not prov_media.available: + continue # get streamdetails from provider music_prov = self.mass.get_provider(prov_media.provider) if not music_prov or not music_prov.available: @@ -1125,150 +543,243 @@ class MusicManager: return streamdetails return None - ################ Library synchronization logic ################ + ################ ADD MediaItem(s) to database helpers ################ - @run_periodic(3600 * 3) - async def __async_music_providers_sync(self): - """Periodic sync of all music providers.""" - await asyncio.sleep(10) - for prov in self.mass.get_providers(ProviderType.MUSIC_PROVIDER): - await self.async_music_provider_sync(prov.id) + async def async_add_artist(self, artist: Artist) -> int: + """Add artist to local db and return the database item.""" + if not artist.musicbrainz_id: + artist.musicbrainz_id = await self.__async_get_artist_musicbrainz_id(artist) + # grab additional metadata + artist.metadata = await self.mass.metadata.async_get_artist_metadata( + artist.musicbrainz_id, artist.metadata + ) + db_item = await self.mass.database.async_add_artist(artist) + # also fetch same artist on all providers + self.mass.add_background_task(self.async_match_artist(db_item)) + return db_item - async def async_music_provider_sync(self, prov_id: str): + async def async_add_album(self, album: Album) -> int: + """Add album to local db and return the database item.""" + # make sure we have an artist + assert album.artist + db_item = await self.mass.database.async_add_album(album) + # also fetch same album on all providers + self.mass.add_background_task(self.async_match_album(db_item)) + return db_item + + async def async_add_track(self, track: Track) -> int: + """Add track to local db and return the new database id.""" + # make sure we have artists + assert track.artists + # make sure we have an album + assert track.album or track.albums + db_item = await self.mass.database.async_add_track(track) + # also fetch same track on all providers (will also get other quality versions) + self.mass.add_background_task(self.async_match_track(db_item)) + return db_item + + async def __async_get_artist_musicbrainz_id(self, artist: Artist): + """Fetch musicbrainz id by performing search using the artist name, albums and tracks.""" + # try with album first + for lookup_album in await self.async_get_artist_albums( + artist.item_id, artist.provider + ): + if not lookup_album: + continue + musicbrainz_id = await self.musicbrainz.async_get_mb_artist_id( + artist.name, + albumname=lookup_album.name, + album_upc=lookup_album.upc, + ) + if musicbrainz_id: + return musicbrainz_id + # fallback to track + for lookup_track in await self.async_get_artist_toptracks( + artist.item_id, artist.provider + ): + if not lookup_track: + continue + musicbrainz_id = await self.musicbrainz.async_get_mb_artist_id( + artist.name, + trackname=lookup_track.name, + track_isrc=lookup_track.isrc, + ) + if musicbrainz_id: + return musicbrainz_id + # lookup failed, use the shitty workaround to use the name as id. + LOGGER.warning("Unable to get musicbrainz ID for artist %s !", artist.name) + return artist.name + + async def async_match_artist(self, db_artist: Artist): """ - Sync a music provider. + Try to find matching artists on all providers for the provided (database) artist_id. - param prov_id: {string} -- provider id to sync + This is used to link objects of different providers together. """ - provider = self.mass.get_provider(prov_id) - if not provider: - return - if MediaType.Album in provider.supported_mediatypes: - await self.async_library_albums_sync(prov_id) - if MediaType.Track in provider.supported_mediatypes: - await self.async_library_tracks_sync(prov_id) - if MediaType.Artist in provider.supported_mediatypes: - await self.async_library_artists_sync(prov_id) - if MediaType.Playlist in provider.supported_mediatypes: - await self.async_library_playlists_sync(prov_id) - if MediaType.Radio in provider.supported_mediatypes: - await self.async_library_radios_sync(prov_id) + assert ( + db_artist.provider == "database" + ), "Matching only supported for database items!" + cur_providers = [item.provider for item in db_artist.provider_ids] + for provider in self.mass.get_providers(ProviderType.MUSIC_PROVIDER): + if provider.id in cur_providers: + continue + if Artist not in provider.supported_mediatypes: + continue + if not await self.__async_match_prov_artist(db_artist, provider): + LOGGER.debug( + "Could not find match for Artist %s on provider %s", + db_artist.name, + provider.name, + ) - @sync_task("artists") - async def async_library_artists_sync(self, provider_id: str): - """Sync library artists for given provider.""" - music_provider = self.mass.get_provider(provider_id) - cache_key = f"library_artists_{provider_id}" - prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) - cur_db_ids = [] - for item in await music_provider.async_get_library_artists(): - db_item = await self.async_get_artist(item.item_id, provider_id, lazy=False) - cur_db_ids.append(db_item.item_id) - await self.mass.database.async_add_to_library( - db_item.item_id, MediaType.Artist, provider_id + async def __async_match_prov_artist( + self, db_artist: Artist, provider: MusicProvider + ): + """Try to find matching artists on given provider for the provided (database) artist.""" + LOGGER.debug( + "Trying to match artist %s on provider %s", db_artist.name, provider.name + ) + # try to get a match with some reference albums of this artist + for ref_album in await self.async_get_artist_albums( + db_artist.item_id, db_artist.provider + ): + searchstr = "%s - %s" % (db_artist.name, ref_album.name) + search_result = await self.async_search_provider( + searchstr, provider.id, [MediaType.Album], limit=10 ) - # process deletions - for db_id in prev_db_ids: - if db_id not in cur_db_ids: - await self.mass.database.async_remove_from_library( - db_id, MediaType.Artist, provider_id - ) - # store ids in cache for next sync - await self.mass.cache.async_set(cache_key, cur_db_ids) + for search_result_item in search_result.albums: + if compare_album(search_result_item, ref_album): + # 100% album match, we can simply update the db with the provider id + await self.mass.database.async_update_artist( + db_artist.item_id, search_result_item.artist + ) + return True - @sync_task("albums") - async def async_library_albums_sync(self, provider_id: str): - """Sync library albums for given provider.""" - music_provider = self.mass.get_provider(provider_id) - cache_key = f"library_albums_{provider_id}" - prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) - cur_db_ids = [] - for item in await music_provider.async_get_library_albums(): + # try to get a match with some reference tracks of this artist + for ref_track in await self.async_get_artist_toptracks( + db_artist.item_id, db_artist.provider + ): + searchstr = "%s - %s" % (db_artist.name, ref_track.name) + search_results = await self.async_search_provider( + searchstr, provider.id, [MediaType.Track], limit=10 + ) + for search_result_item in search_results.tracks: + if compare_track(search_result_item, ref_track): + # get matching artist from track + for search_item_artist in search_result_item.artists: + if compare_strings(db_artist.name, search_item_artist.name): + # 100% match, we can simply update the db with additional provider ids + await self.mass.database.async_update_artist( + db_artist.item_id, search_item_artist + ) + return True + return False - db_album = await self.async_get_album( - item.item_id, provider_id, album_details=item, lazy=False + async def async_match_album(self, db_album: Album): + """ + Try to find matching album on all providers for the provided (database) album_id. + + This is used to link objects of different providers/qualities together. + """ + assert ( + db_album.provider == "database" + ), "Matching only supported for database items!" + if not isinstance(db_album, FullAlbum): + # matching only works if we have a full album object + db_album = await self.mass.database.async_get_album(db_album.item_id) + + async def find_prov_match(provider): + LOGGER.debug( + "Trying to match album %s on provider %s", db_album.name, provider.name ) - cur_db_ids.append(db_album.item_id) - await self.mass.database.async_add_to_library( - db_album.item_id, MediaType.Album, provider_id + match_found = False + searchstr = "%s - %s" % (db_album.artist.name, db_album.name) + if db_album.version: + searchstr += " " + db_album.version + search_result = await self.async_search_provider( + searchstr, provider.id, [MediaType.Album], limit=5 ) - # process deletions - for db_id in prev_db_ids: - if db_id not in cur_db_ids: - await self.mass.database.async_remove_from_library( - db_id, MediaType.Album, provider_id + for search_result_item in search_result.albums: + if not search_result_item.available: + continue + if compare_album(search_result_item, db_album): + # 100% match, we can simply update the db with additional provider ids + await self.mass.database.async_update_album( + db_album.item_id, search_result_item + ) + match_found = True + # no match found + if not match_found: + LOGGER.debug( + "Could not find match for Album %s on provider %s", + db_album.name, + provider.name, ) - # store ids in cache for next sync - await self.mass.cache.async_set(cache_key, cur_db_ids) - @sync_task("tracks") - async def async_library_tracks_sync(self, provider_id: str): - """Sync library tracks for given provider.""" - music_provider = self.mass.get_provider(provider_id) - cache_key = f"library_tracks_{provider_id}" - prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) - cur_db_ids = [] - for item in await music_provider.async_get_library_tracks(): - db_item = await self.async_get_track( - item.item_id, provider_id=provider_id, lazy=False + # try to find match on all providers + providers = self.mass.get_providers(ProviderType.MUSIC_PROVIDER) + for provider in providers: + if Album in provider.supported_mediatypes: + await find_prov_match(provider) + + async def async_match_track(self, db_track: Track): + """ + Try to find matching track on all providers for the provided (database) track_id. + + This is used to link objects of different providers/qualities together. + """ + assert ( + db_track.provider == "database" + ), "Matching only supported for database items!" + if not isinstance(db_track, FullTrack): + # matching only works if we have a full track object + db_track = await self.mass.database.async_get_track(db_track.item_id) + for provider in self.mass.get_providers(ProviderType.MUSIC_PROVIDER): + if Track not in provider.supported_mediatypes: + continue + LOGGER.debug( + "Trying to match track %s on provider %s", db_track.name, provider.name ) - cur_db_ids.append(db_item.item_id) - if db_item.item_id not in prev_db_ids: - await self.mass.database.async_add_to_library( - db_item.item_id, MediaType.Track, provider_id - ) - # process deletions - for db_id in prev_db_ids: - if db_id not in cur_db_ids: - await self.mass.database.async_remove_from_library( - db_id, MediaType.Track, provider_id + match_found = False + for db_track_artist in db_track.artists: + if match_found: + break + searchstr = "%s - %s" % (db_track_artist.name, db_track.name) + if db_track.version: + searchstr += " " + db_track.version + search_result = await self.async_search_provider( + searchstr, provider.id, [MediaType.Track], limit=10 ) - # store ids in cache for next sync - await self.mass.cache.async_set(cache_key, cur_db_ids) + for search_result_item in search_result.tracks: + if not search_result_item.available: + continue + if compare_track(search_result_item, db_track): + # 100% match, we can simply update the db with additional provider ids + match_found = True + await self.mass.database.async_update_track( + db_track.item_id, search_result_item + ) - @sync_task("playlists") - async def async_library_playlists_sync(self, provider_id: str): - """Sync library playlists for given provider.""" - music_provider = self.mass.get_provider(provider_id) - cache_key = f"library_playlists_{provider_id}" - prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) - cur_db_ids = [] - for playlist in await music_provider.async_get_library_playlists(): - # always add to db because playlist attributes could have changed - db_item = await self.mass.database.async_add_playlist(playlist) - cur_db_ids.append(db_item.item_id) - await self.mass.database.async_add_to_library( - db_item.item_id, MediaType.Playlist, playlist.provider - ) - # precache playlist tracks - await self.async_get_playlist_tracks(db_item.item_id, db_item.provider) - # process playlist deletions - for db_id in prev_db_ids: - if db_id not in cur_db_ids: - await self.mass.database.async_remove_from_library( - db_id, MediaType.Playlist, provider_id + if not match_found: + LOGGER.debug( + "Could not find match for Track %s on provider %s", + db_track.name, + provider.name, ) - # store ids in cache for next sync - await self.mass.cache.async_set(cache_key, cur_db_ids) - @sync_task("radios") - async def async_library_radios_sync(self, provider_id: str): - """Sync library radios for given provider.""" - music_provider = self.mass.get_provider(provider_id) - cache_key = f"library_radios_{provider_id}" - prev_db_ids = await self.mass.cache.async_get(cache_key, default=[]) - cur_db_ids = [] - for item in await music_provider.async_get_library_radios(): - db_radio = await self.async_get_radio(item.item_id, provider_id) - cur_db_ids.append(db_radio.item_id) - await self.mass.database.async_add_to_library( - db_radio.item_id, MediaType.Radio, provider_id + async def __get_provider_id(self, media_item: MediaItem) -> tuple: + """Return provider and item id.""" + if media_item.provider == "database": + media_item = await self.mass.database.async_get_item_by_prov_id( + "database", media_item.item_id, media_item.media_type ) - # process deletions - for db_id in prev_db_ids: - if db_id not in cur_db_ids: - await self.mass.database.async_remove_from_library( - db_id, MediaType.Radio, provider_id - ) - # store ids in cache for next sync - await self.mass.cache.async_set(cache_key, cur_db_ids) + for prov in media_item.provider_ids: + if prov.available and self.mass.get_provider(prov.provider): + provider = self.mass.get_provider(prov.provider) + if provider and provider.available: + return (prov.provider, prov.item_id) + else: + provider = self.mass.get_provider(media_item.provider) + if provider and provider.available: + return (media_item.provider, media_item.item_id) + return None, None diff --git a/music_assistant/managers/players.py b/music_assistant/managers/players.py index 739776f0..cf51dab7 100755 --- a/music_assistant/managers/players.py +++ b/music_assistant/managers/players.py @@ -15,7 +15,7 @@ from music_assistant.constants import ( ) from music_assistant.helpers.typing import MusicAssistantType from music_assistant.helpers.util import callback, run_periodic, try_parse_int -from music_assistant.models.media_types import MediaItem, MediaType, Track +from music_assistant.models.media_types import MediaItem, MediaType from music_assistant.models.player import ( PlaybackState, Player, @@ -277,7 +277,7 @@ class PlayerManager: for track in tracks: if not track.available: continue - queue_item = QueueItem(track) + queue_item = QueueItem.from_track(track) # generate uri for this queue item queue_item.uri = "%s/stream/queue/%s/%s" % ( self.mass.web.url, @@ -314,13 +314,7 @@ class PlayerManager: QueueOption.Next -> Play item(s) after current playing item QueueOption.Add -> Append new items at end of the queue """ - queue_item = QueueItem( - Track( - item_id=uri, - provider="uri", - name=uri, - ) - ) + queue_item = QueueItem(item_id=uri, provider="uri", name=uri) # generate uri for this queue item queue_item.uri = "%s/stream/%s/%s" % ( self.mass.web.url, diff --git a/music_assistant/mass.py b/music_assistant/mass.py index 92cab7e5..e22ddea7 100644 --- a/music_assistant/mass.py +++ b/music_assistant/mass.py @@ -6,7 +6,7 @@ import importlib import logging import os import threading -from typing import Any, Awaitable, Callable, Dict, List, Optional, Union +from typing import Any, Awaitable, Callable, Coroutine, Dict, List, Optional, Union import aiohttp from music_assistant.constants import ( @@ -16,9 +16,11 @@ from music_assistant.constants import ( EVENT_SHUTDOWN, ) from music_assistant.helpers.cache import Cache +from music_assistant.helpers.migration import check_migrations from music_assistant.helpers.util import callback, get_ip_pton, is_callback from music_assistant.managers.config import ConfigManager from music_assistant.managers.database import DatabaseManager +from music_assistant.managers.library import LibraryManager from music_assistant.managers.metadata import MetaDataManager from music_assistant.managers.music import MusicManager from music_assistant.managers.players import PlayerManager @@ -54,6 +56,7 @@ class MusicAssistant: self._http_session = None self._event_listeners = [] self._providers = {} + self._background_tasks = None # init core managers/controllers self._config = ConfigManager(self, datapath) @@ -62,6 +65,7 @@ class MusicAssistant: self._metadata = MetaDataManager(self) self._web = WebServer(self, port) self._music = MusicManager(self) + self._library = LibraryManager(self) self._players = PlayerManager(self) self._streams = StreamManager(self) # shared zeroconf instance @@ -78,13 +82,16 @@ class MusicAssistant: loop=self.loop, connector=aiohttp.TCPConnector(enable_cleanup_closed=True, ssl=False), ) - await self._database.async_setup() + # run migrations if needed + await check_migrations(self) await self._cache.async_setup() await self._music.async_setup() await self._players.async_setup() await self.__async_preload_providers() await self.__async_setup_discovery() await self._web.async_setup() + await self._library.async_setup() + self.loop.create_task(self.__process_background_tasks()) async def async_stop(self): """Stop running the music assistant server.""" @@ -119,6 +126,11 @@ class MusicAssistant: """Return the Music controller/manager.""" return self._music + @property + def library(self) -> LibraryManager: + """Return the Library controller/manager.""" + return self._library + @property def config(self) -> ConfigManager: """Return the Configuration controller/manager.""" @@ -249,6 +261,16 @@ class MusicAssistant: return remove_listener + @callback + def add_background_task(self, task: Coroutine): + """Add a coroutine/task to the end of the job queue. + + target: target to call. + args: parameters for method to call. + """ + if self._background_tasks: + self._background_tasks.put_nowait(task) + @callback def add_job( self, target: Callable[..., Any], *args: Any, **kwargs: Any @@ -292,6 +314,14 @@ class MusicAssistant: task = self.loop.run_in_executor(None, target, *args, *kwargs) # type: ignore return task + async def __process_background_tasks(self): + """Background tasks that takes care of slowly handling jobs in the queue.""" + self._background_tasks = asyncio.Queue() + while not self.exit: + task = await self._background_tasks.get() + await task + await asyncio.sleep(1) + async def __async_setup_discovery(self) -> None: """Make this Music Assistant instance discoverable on the network.""" zeroconf_type = "_music-assistant._tcp.local." diff --git a/music_assistant/models/media_types.py b/music_assistant/models/media_types.py index 1f60367a..7543e95d 100755 --- a/music_assistant/models/media_types.py +++ b/music_assistant/models/media_types.py @@ -5,8 +5,8 @@ from enum import Enum, IntEnum from typing import Any, List, Mapping import ujson +import unidecode from mashumaro import DataClassDictMixin -from music_assistant.helpers.util import get_sort_name class MediaType(Enum): @@ -70,24 +70,29 @@ class MediaItem(DataClassDictMixin): metadata: Any = field(default_factory=dict) provider_ids: List[MediaItemProviderId] = field(default_factory=list) in_library: bool = False - is_lazy: bool = False @classmethod def from_db_row(cls, db_row: Mapping): """Create MediaItem object from database row.""" db_row = dict(db_row) - for key in ["artists", "artist", "album", "metadata", "provider_ids"]: + for key in ["artists", "artist", "album", "metadata", "provider_ids", "albums"]: if key in db_row: db_row[key] = ujson.loads(db_row[key]) db_row["provider"] = "database" if "in_library" in db_row: db_row["in_library"] = bool(db_row["in_library"]) + if db_row.get("albums"): + db_row["album"] = db_row["albums"][0] return cls.from_dict(db_row) @property def sort_name(self): """Return sort name.""" - return get_sort_name(self.name) + sort_name = self.name + for item in ["The ", "De ", "de ", "Les "]: + if self.name.startswith(item): + sort_name = "".join(self.name.split(item)[1:]) + return unidecode.unidecode(sort_name).lower() @property def available(self): @@ -106,14 +111,19 @@ class Artist(MediaItem): @dataclass -class AlbumArtist(DataClassDictMixin): - """Representation of a minimized artist object.""" +class ItemMapping(DataClassDictMixin): + """Representation of a minimized item object.""" item_id: str = "" provider: str = "" name: str = "" media_type: MediaType = MediaType.Artist + @classmethod + def from_item(cls, item: Mapping): + """Create ItemMapping object from regular item.""" + return cls.from_dict(item.to_dict()) + @dataclass class Album(MediaItem): @@ -122,29 +132,16 @@ class Album(MediaItem): media_type: MediaType = MediaType.Album version: str = "" year: int = 0 - artist: AlbumArtist = None + artist: ItemMapping = None album_type: AlbumType = AlbumType.Album upc: str = "" @dataclass -class TrackArtist(DataClassDictMixin): - """Representation of a minimized artist object.""" +class FullAlbum(Album): + """Model for an album with full details.""" - item_id: str = "" - provider: str = "" - name: str = "" - media_type: MediaType = MediaType.Artist - - -@dataclass -class TrackAlbum(DataClassDictMixin): - """Representation of a minimized album object.""" - - item_id: str = "" - provider: str = "" - name: str = "" - media_type: MediaType = MediaType.Album + artist: Artist = None @dataclass @@ -154,12 +151,24 @@ class Track(MediaItem): media_type: MediaType = MediaType.Track duration: int = 0 version: str = "" - artists: List[TrackArtist] = field(default_factory=list) - album: TrackAlbum = None - disc_number: int = 1 - track_number: int = 1 - position: int = 0 isrc: str = "" + artists: List[ItemMapping] = field(default_factory=list) + albums: List[ItemMapping] = field(default_factory=list) + # album track only + album: ItemMapping = None + disc_number: int = 0 + track_number: int = 0 + # playlist track only + position: int = 0 + + +@dataclass +class FullTrack(Track): + """Model for an album with full details.""" + + artists: List[Artist] = field(default_factory=list) + albums: List[Album] = field(default_factory=list) + album: Album = None @dataclass diff --git a/music_assistant/models/player_queue.py b/music_assistant/models/player_queue.py index 3924f06d..f3290650 100755 --- a/music_assistant/models/player_queue.py +++ b/music_assistant/models/player_queue.py @@ -6,7 +6,7 @@ import time import uuid from dataclasses import dataclass from enum import Enum -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union from music_assistant.constants import ( CONF_CROSSFADE_DURATION, @@ -21,7 +21,7 @@ from music_assistant.helpers.typing import ( PlayerType, ) from music_assistant.helpers.util import callback -from music_assistant.models.media_types import Track +from music_assistant.models.media_types import Radio, Track from music_assistant.models.player import PlaybackState, PlayerFeature from music_assistant.models.streamdetails import StreamDetails @@ -49,14 +49,14 @@ class QueueItem(Track): uri: str = "" queue_item_id: str = "" - def __init__(self, media_item=None) -> None: - """Initialize class.""" - super().__init__() + def __post_init__(self): + """Generate unique id for the QueueItem.""" self.queue_item_id = str(uuid.uuid4()) - # if existing media_item given, load those values - if media_item: - for key, value in media_item.__dict__.items(): - setattr(self, key, value) + + @classmethod + def from_track(cls, track: Union[Track, Radio]): + """Construct QueueItem from track/raio item.""" + return cls.from_dict(track.to_dict()) class PlayerQueue: @@ -99,7 +99,7 @@ class PlayerQueue: return self._player_id def get_stream_url(self) -> str: - """Return the full stream url for this QueueStream.""" + """Return the full stream url for the player's Queue Stream.""" uri = f"{self.mass.web.url}/stream/queue/{self.player_id}" # we set the checksum just to invalidate cache stuf uri += f"?checksum={time.time()}" diff --git a/music_assistant/providers/chromecast/player.py b/music_assistant/providers/chromecast/player.py index 7349f73f..723e6bf6 100644 --- a/music_assistant/providers/chromecast/player.py +++ b/music_assistant/providers/chromecast/player.py @@ -6,8 +6,9 @@ from typing import List, Optional import pychromecast from asyncio_throttle import Throttler +from music_assistant.helpers.compare import compare_strings from music_assistant.helpers.typing import MusicAssistantType -from music_assistant.helpers.util import async_yield_chunks, compare_strings +from music_assistant.helpers.util import async_yield_chunks from music_assistant.models.config_entry import ConfigEntry from music_assistant.models.player import ( DeviceInfo, @@ -390,9 +391,7 @@ class ChromecastPlayer(Player): player_queue = self.mass.players.get_player_queue(self.player_id) if player_queue.use_queue_stream: # create CC queue so that skip and previous will work - queue_item = QueueItem() - queue_item.name = "Music Assistant" - queue_item.uri = uri + queue_item = QueueItem(name="Music Assistant", uri=uri) return await self.async_cmd_queue_load([queue_item, queue_item]) await self.__async_try_chromecast_command( self._chromecast.play_media, uri, "audio/flac" diff --git a/music_assistant/providers/file/__init__.py b/music_assistant/providers/file/__init__.py index 45d96452..75fca69a 100644 --- a/music_assistant/providers/file/__init__.py +++ b/music_assistant/providers/file/__init__.py @@ -395,9 +395,7 @@ class FileProvider(MusicProvider): prov_id = uri.split("://")[0] prov_item_id = uri.split("/")[-1].split(".")[0].split(":")[-1] try: - return await self.mass.music.async_get_track( - prov_item_id, prov_id, lazy=False - ) + return await self.mass.music.async_get_track(prov_item_id, prov_id) except Exception as exc: LOGGER.warning("Could not parse uri %s to track: %s", uri, str(exc)) return None diff --git a/music_assistant/providers/spotify/__init__.py b/music_assistant/providers/spotify/__init__.py index 4da742ac..1053b468 100644 --- a/music_assistant/providers/spotify/__init__.py +++ b/music_assistant/providers/spotify/__init__.py @@ -133,25 +133,29 @@ class SpotifyProvider(MusicProvider): searchresult = await self.__async_get_data("search", params=params) if searchresult: if "artists" in searchresult: - for item in searchresult["artists"]["items"]: - artist = await self.__async_parse_artist(item) - if artist: - result.artists.append(artist) + result.artists = [ + await self.__async_parse_artist(item) + for item in searchresult["artists"]["items"] + if (item and item["id"]) + ] if "albums" in searchresult: - for item in searchresult["albums"]["items"]: - album = await self.__async_parse_album(item) - if album: - result.albums.append(album) + result.albums = [ + await self.__async_parse_album(item) + for item in searchresult["albums"]["items"] + if (item and item["id"]) + ] if "tracks" in searchresult: - for item in searchresult["tracks"]["items"]: - track = await self.__async_parse_track(item) - if track: - result.tracks.append(track) + result.tracks = [ + await self.__async_parse_track(item) + for item in searchresult["tracks"]["items"] + if (item and item["id"]) + ] if "playlists" in searchresult: - for item in searchresult["playlists"]["items"]: - playlist = await self.__async_parse_playlist(item) - if playlist: - result.playlists.append(playlist) + result.playlists = [ + await self.__async_parse_playlist(item) + for item in searchresult["playlists"]["items"] + if (item and item["id"]) + ] return result async def async_get_library_artists(self) -> List[Artist]: @@ -196,22 +200,22 @@ class SpotifyProvider(MusicProvider): async def async_get_artist(self, prov_artist_id) -> Artist: """Get full artist details by id.""" artist_obj = await self.__async_get_data("artists/%s" % prov_artist_id) - return await self.__async_parse_artist(artist_obj) + return await self.__async_parse_artist(artist_obj) if artist_obj else None async def async_get_album(self, prov_album_id) -> Album: """Get full album details by id.""" album_obj = await self.__async_get_data("albums/%s" % prov_album_id) - return await self.__async_parse_album(album_obj) + return await self.__async_parse_album(album_obj) if album_obj else None async def async_get_track(self, prov_track_id) -> Track: """Get full track details by id.""" track_obj = await self.__async_get_data("tracks/%s" % prov_track_id) - return await self.__async_parse_track(track_obj) + return await self.__async_parse_track(track_obj) if track_obj else None async def async_get_playlist(self, prov_playlist_id) -> Playlist: """Get full playlist details by id.""" playlist_obj = await self.__async_get_data(f"playlists/{prov_playlist_id}") - return await self.__async_parse_playlist(playlist_obj) + return await self.__async_parse_playlist(playlist_obj) if playlist_obj else None async def async_get_album_tracks(self, prov_album_id) -> List[Track]: """Get all album tracks for given album id.""" @@ -407,7 +411,7 @@ class SpotifyProvider(MusicProvider): track.artists.append(artist) for track_artist in track_obj.get("artists", []): artist = await self.__async_parse_artist(track_artist) - if artist: + if artist and artist.item_id not in [x.item_id for x in track.artists]: track.artists.append(artist) track.name, track.version = parse_title_and_version(track_obj["name"]) track.metadata["explicit"] = str(track_obj["explicit"]).lower() @@ -468,7 +472,7 @@ class SpotifyProvider(MusicProvider): LOGGER.info("Succesfully logged in to Spotify as %s", self.sp_user["id"]) self.__auth_token = tokeninfo else: - raise Exception("Can't get Spotify token for user %s" % self._username) + LOGGER.error("Login failed for user %s", self._username) return tokeninfo async def __async_get_token(self): diff --git a/music_assistant/providers/tunein/__init__.py b/music_assistant/providers/tunein/__init__.py index dc170924..25e3792d 100644 --- a/music_assistant/providers/tunein/__init__.py +++ b/music_assistant/providers/tunein/__init__.py @@ -94,7 +94,7 @@ class TuneInProvider(MusicProvider): # TODO: search for radio stations return result - async def async_get_radios(self) -> List[Radio]: + async def async_get_library_radios(self) -> List[Radio]: """Retrieve library/subscribed radio stations from the provider.""" params = {"c": "presets"} result = await self.__async_get_data("Browse.ashx", params) diff --git a/music_assistant/web/__init__.py b/music_assistant/web/__init__.py index 40215c98..39115248 100755 --- a/music_assistant/web/__init__.py +++ b/music_assistant/web/__init__.py @@ -42,7 +42,8 @@ class WebServer: self.mass = mass self._port = port # load/create/update config - self._local_ip = get_ip() + # self._hostname = get_hostname() or get_ip() + self._hostname = get_ip() self._device_id = f"{uuid.getnode()}_{get_hostname()}" self.config = mass.config.base["web"] self._runner = None @@ -107,7 +108,7 @@ class WebServer: @property def host(self): """Return the local IP address/host for this Music Assistant instance.""" - return self._local_ip + return self._hostname @property def port(self): diff --git a/music_assistant/web/endpoints/albums.py b/music_assistant/web/endpoints/albums.py index f0c68bd4..d7abacb6 100644 --- a/music_assistant/web/endpoints/albums.py +++ b/music_assistant/web/endpoints/albums.py @@ -22,11 +22,10 @@ async def async_album(request: Request): """Get full album details.""" item_id = request.match_info.get("item_id") provider = request.rel_url.query.get("provider") - lazy = request.rel_url.query.get("lazy", "true") != "false" if item_id is None or provider is None: return Response(text="invalid item or provider", status=501) return await async_json_response( - await request.app["mass"].music.async_get_album(item_id, provider, lazy=lazy) + await request.app["mass"].music.async_get_album(item_id, provider) ) diff --git a/music_assistant/web/endpoints/artists.py b/music_assistant/web/endpoints/artists.py index 1847dff2..b165ca1b 100644 --- a/music_assistant/web/endpoints/artists.py +++ b/music_assistant/web/endpoints/artists.py @@ -21,12 +21,9 @@ async def async_artist(request: Request): """Get full artist details.""" item_id = request.match_info.get("item_id") provider = request.rel_url.query.get("provider") - lazy = request.rel_url.query.get("lazy", "true") != "false" if item_id is None or provider is None: return Response(text="invalid item or provider", status=501) - result = await request.app["mass"].music.async_get_artist( - item_id, provider, lazy=lazy - ) + result = await request.app["mass"].music.async_get_artist(item_id, provider) return await async_json_response(result) diff --git a/music_assistant/web/endpoints/images.py b/music_assistant/web/endpoints/images.py index a388bae4..7f4c58cc 100644 --- a/music_assistant/web/endpoints/images.py +++ b/music_assistant/web/endpoints/images.py @@ -1,15 +1,17 @@ """Images API endpoints.""" - import os +from io import BytesIO from aiohttp.web import FileResponse, Request, Response, RouteTableDef +from music_assistant.helpers.typing import MusicAssistantType from music_assistant.models.media_types import MediaType +from PIL import Image routes = RouteTableDef() -@routes.get("/api/providers/{provider_id}/icon") +@routes.get("/api/images/provider-icon/{provider_id}") async def async_get_provider_icon(request: Request): """Get Provider icon.""" provider_id = request.match_info.get("provider_id") @@ -21,20 +23,92 @@ async def async_get_provider_icon(request: Request): return Response(status=404) -@routes.get("/api/{media_type}/{media_id}/thumb") -async def async_get_image(request: Request): +@routes.get("/api/images/thumb") +async def async_get_image_thumb(request: Request): """Get (resized) thumb image.""" - media_type_str = request.match_info.get("media_type") - media_type = MediaType(media_type_str) - media_id = request.match_info.get("media_id") - provider = request.rel_url.query.get("provider") - if media_id is None or provider is None: - return Response(text="invalid media_id or provider", status=501) + mass = request.app["mass"] size = int(request.rel_url.query.get("size", 0)) - img_file = await request.app["mass"].music.async_get_image_thumb( - media_id, provider, media_type, size - ) + provider = request.rel_url.query.get("provider") + item_id = request.rel_url.query.get("item_id") + + if provider and item_id: + media_type = MediaType(request.rel_url.query.get("media_type")) + url = await async_get_image_url(mass, item_id, provider, media_type) + else: + url = request.rel_url.query.get("url") + if not url: + return Response(status=404, text="Invalid URL OR media details given") + + img_file = await async_get_image_file(mass, url, size) if not img_file or not os.path.isfile(img_file): return Response(status=404) headers = {"Cache-Control": "max-age=86400, public", "Pragma": "public"} return FileResponse(img_file, headers=headers) + + +async def async_get_image_file(mass: MusicAssistantType, url, size: int = 150): + """Get path to (resized) thumbnail image for given image url.""" + cache_folder = os.path.join(mass.config.data_path, ".thumbs") + cache_id = await mass.database.async_get_thumbnail_id(url, size) + cache_file = os.path.join(cache_folder, f"{cache_id}.png") + if os.path.isfile(cache_file): + # return file from cache + return cache_file + # no file in cache so we should get it + os.makedirs(cache_folder, exist_ok=True) + # download base image + async with mass.http_session.get(url, verify_ssl=False) as response: + assert response.status == 200 + img_data = BytesIO(await response.read()) + + # save resized image + if size: + basewidth = size + img = Image.open(img_data) + wpercent = basewidth / float(img.size[0]) + hsize = int((float(img.size[1]) * float(wpercent))) + img = img.resize((basewidth, hsize), Image.ANTIALIAS) + img.save(cache_file) + else: + with open(cache_file, "wb") as _file: + _file.write(img_data.getvalue()) + # return file from cache + return cache_file + + +async def async_get_image_url( + mass: MusicAssistantType, item_id: str, provider_id: str, media_type: MediaType +): + """Get url to image for given media item.""" + item = await mass.music.async_get_item(item_id, provider_id, media_type) + if not item: + return None + if item and item.metadata.get("image"): + return item.metadata["image"] + if ( + hasattr(item, "album") + and hasattr(item.album, "metadata") + and item.album.metadata.get("image") + ): + return item.album.metadata["image"] + if hasattr(item, "albums"): + for album in item.albums: + if hasattr(album, "metadata") and album.metadata.get("image"): + return album.metadata["image"] + if ( + hasattr(item, "artist") + and hasattr(item.artist, "metadata") + and item.artist.metadata.get("image") + ): + return item.album.metadata["image"] + if media_type == MediaType.Track and item.album: + # try album instead for tracks + return await async_get_image_url( + mass, item.album.item_id, item.album.provider, MediaType.Album + ) + elif media_type == MediaType.Album and item.artist: + # try artist instead for albums + return await async_get_image_url( + mass, item.artist.item_id, item.artist.provider, MediaType.Artist + ) + return None diff --git a/music_assistant/web/endpoints/library.py b/music_assistant/web/endpoints/library.py index c0e217e1..32f6caea 100644 --- a/music_assistant/web/endpoints/library.py +++ b/music_assistant/web/endpoints/library.py @@ -14,7 +14,7 @@ async def async_library_artists(request: Request): orderby = request.query.get("orderby", "name") return await async_json_response( - await request.app["mass"].music.async_get_library_artists(orderby=orderby) + await request.app["mass"].library.async_get_library_artists(orderby=orderby) ) @@ -25,7 +25,7 @@ async def async_library_albums(request: Request): orderby = request.query.get("orderby", "name") return await async_json_response( - await request.app["mass"].music.async_get_library_albums(orderby=orderby) + await request.app["mass"].library.async_get_library_albums(orderby=orderby) ) @@ -36,7 +36,7 @@ async def async_library_tracks(request: Request): orderby = request.query.get("orderby", "name") return await async_json_response( - await request.app["mass"].music.async_get_library_tracks(orderby=orderby) + await request.app["mass"].library.async_get_library_tracks(orderby=orderby) ) @@ -47,7 +47,7 @@ async def async_library_radios(request: Request): orderby = request.query.get("orderby", "name") return await async_json_response( - await request.app["mass"].music.async_get_library_radios(orderby=orderby) + await request.app["mass"].library.async_get_library_radios(orderby=orderby) ) @@ -58,7 +58,7 @@ async def async_library_playlists(request: Request): orderby = request.query.get("orderby", "name") return await async_json_response( - await request.app["mass"].music.async_get_library_playlists(orderby=orderby) + await request.app["mass"].library.async_get_library_playlists(orderby=orderby) ) @@ -68,7 +68,7 @@ async def async_library_add(request: Request): """Add item(s) to the library.""" body = await request.json() media_items = await async_media_items_from_body(request.app["mass"], body) - result = await request.app["mass"].music.async_library_add(media_items) + result = await request.app["mass"].library.async_library_add(media_items) return await async_json_response(result) @@ -78,5 +78,5 @@ async def async_library_remove(request: Request): """Remove item(s) from the library.""" body = await request.json() media_items = await async_media_items_from_body(request.app["mass"], body) - result = await request.app["mass"].music.async_library_remove(media_items) + result = await request.app["mass"].library.async_library_remove(media_items) return await async_json_response(result) diff --git a/music_assistant/web/endpoints/streams.py b/music_assistant/web/endpoints/streams.py index 33bb301c..62e110eb 100644 --- a/music_assistant/web/endpoints/streams.py +++ b/music_assistant/web/endpoints/streams.py @@ -25,10 +25,13 @@ async def stream_media(request: Request): resp = StreamResponse( status=200, reason="OK", headers={"Content-Type": f"audio/{content_type}"} ) + + resp.enable_chunked_encoding() + resp.enable_compression() await resp.prepare(request) # stream track - async for audio_chunk in request.app["mass"].streams.async_get_stream( + async for audio_chunk in request.app["mass"].streams.async_get_media_stream( streamdetails ): await resp.write(audio_chunk) @@ -48,6 +51,7 @@ async def stream_queue(request: Request): status=200, reason="OK", headers={"Content-Type": "audio/flac"} ) resp.enable_chunked_encoding() + resp.enable_compression() await resp.prepare(request) # stream queue @@ -69,6 +73,8 @@ async def stream_queue_item(request: Request): resp = StreamResponse( status=200, reason="OK", headers={"Content-Type": "audio/flac"} ) + resp.enable_chunked_encoding() + resp.enable_compression() await resp.prepare(request) async for audio_chunk in request.app["mass"].streams.async_stream_queue_item( diff --git a/music_assistant/web/endpoints/tracks.py b/music_assistant/web/endpoints/tracks.py index 110c018a..ec1aa8ae 100644 --- a/music_assistant/web/endpoints/tracks.py +++ b/music_assistant/web/endpoints/tracks.py @@ -33,10 +33,7 @@ async def async_track(request: Request): """Get full track details.""" item_id = request.match_info.get("item_id") provider = request.rel_url.query.get("provider") - lazy = request.rel_url.query.get("lazy", "true") != "false" if item_id is None or provider is None: return Response(text="invalid item or provider", status=501) - result = await request.app["mass"].music.async_get_track( - item_id, provider, lazy=lazy - ) + result = await request.app["mass"].music.async_get_track(item_id, provider) return await async_json_response(result) diff --git a/music_assistant/web/endpoints/websocket.py b/music_assistant/web/endpoints/websocket.py index 62f51ea2..1e4c9b78 100644 --- a/music_assistant/web/endpoints/websocket.py +++ b/music_assistant/web/endpoints/websocket.py @@ -113,6 +113,54 @@ async def async_websocket_handler(request: Request): return ws_response +@ws_command("players") +async def async_players(mass: MusicAssistantType, msg_details: dict): + """Return players.""" + if msg_details and msg_details.get("player_id"): + return mass.players.get_player_state(msg_details["player_id"]) + return mass.players.player_states + + +@ws_command("tracks") +async def tracks(mass: MusicAssistantType, msg_details: dict): + """Return tracks.""" + if msg_details and msg_details.get("item_id"): + return await mass.music.async_get_track(msg_details["item_id"]) + return await mass.music.async_get_library_tracks() + + +@ws_command("albums") +async def albums(mass: MusicAssistantType, msg_details: dict): + """Return albums.""" + if msg_details and msg_details.get("item_id"): + return await mass.music.async_get_album(msg_details["item_id"]) + return await mass.music.async_get_library_albums() + + +@ws_command("artists") +async def artists(mass: MusicAssistantType, msg_details: dict): + """Return artists.""" + if msg_details and msg_details.get("item_id"): + return await mass.music.async_get_artist(msg_details["item_id"]) + return await mass.music.async_get_library_artists() + + +@ws_command("playlists") +async def playlists(mass: MusicAssistantType, msg_details: dict): + """Return playlists.""" + if msg_details and msg_details.get("item_id"): + return await mass.music.async_get_playlist(msg_details["item_id"]) + return await mass.music.async_get_library_playlists() + + +@ws_command("radios") +async def radios(mass: MusicAssistantType, msg_details: dict): + """Return radios.""" + if msg_details and msg_details.get("item_id"): + return await mass.music.async_get_radio(msg_details["item_id"]) + return await mass.music.async_get_library_radios() + + @ws_command("player_command") async def async_player_command(mass: MusicAssistantType, msg_details: dict): """Handle player command."""