From 010bccd86bcf8267ca2778ce532197c07aa643fb Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Thu, 23 Nov 2023 07:38:36 -0800 Subject: [PATCH] [soundcloud] Improved playlist iteration (#924) * [soundcloud] improved playlist imports Similar to https://github.com/music-assistant/server/pull/922, the playlist import process is incomplete. Through trial and error, I discovered that soundcloud returns all forms of "track lists" as a response to the API call -- and that it was sorted by recency (which explained the behavior I witnessed on `2023.6.0b6`. This PR fixes it right up. It generalizes soundcloud pagination for easier future adoption and for the soundcloud playlists. Manually tested the `asyncsoundcloudpy` API. * added self to manifest codeowners --- .../server/providers/soundcloud/__init__.py | 32 +++++--- .../server/providers/soundcloud/manifest.json | 5 +- .../soundcloudpy/asyncsoundcloudpy.py | 77 ++++++++++++------- 3 files changed, 77 insertions(+), 37 deletions(-) diff --git a/music_assistant/server/providers/soundcloud/__init__.py b/music_assistant/server/providers/soundcloud/__init__.py index 5941cd5b..3346752b 100644 --- a/music_assistant/server/providers/soundcloud/__init__.py +++ b/music_assistant/server/providers/soundcloud/__init__.py @@ -174,21 +174,33 @@ class SoundcloudMusicProvider(MusicProvider): async def get_library_playlists(self) -> AsyncGenerator[Playlist, None]: """Retrieve all library playlists from Soundcloud.""" time_start = time.time() - playlists = await self._soundcloud.get_account_playlists() - self.logger.debug( - "Processing Soundcloud library playlists took %s seconds", - round(time.time() - time_start, 2), - ) - for item in playlists["collection"]: + async for item in self._soundcloud.get_account_playlists(): + try: + raw_playlist = item["playlist"] + except KeyError: + self.logger.debug( + "Unexpected Soundcloud API response when parsing playlists: %s", + item, + ) + continue + try: - playlist_obj = await self._soundcloud.get_playlist_details( - playlist_id=item["playlist"]["id"] + playlist = await self._soundcloud.get_playlist_details( + playlist_id=raw_playlist["id"], ) - yield await self._parse_playlist(playlist_obj) + + yield await self._parse_playlist(playlist) except (KeyError, TypeError, InvalidDataError, IndexError) as error: - self.logger.debug("Parse playlist failed: %s", playlist_obj, exc_info=error) + self.logger.debug( + "Failed to obtain Soundcloud playlist details: %s", raw_playlist, exc_info=error + ) continue + self.logger.debug( + "Processing Soundcloud library playlists took %s seconds", + round(time.time() - time_start, 2), + ) + async def get_library_tracks(self) -> AsyncGenerator[Track, None]: """Retrieve library tracks from Soundcloud.""" time_start = time.time() diff --git a/music_assistant/server/providers/soundcloud/manifest.json b/music_assistant/server/providers/soundcloud/manifest.json index b56ebd1c..4d4378ae 100644 --- a/music_assistant/server/providers/soundcloud/manifest.json +++ b/music_assistant/server/providers/soundcloud/manifest.json @@ -3,7 +3,10 @@ "domain": "soundcloud", "name": "Soundcloud", "description": "Support for the Soundcloud streaming provider in Music Assistant.", - "codeowners": ["@gieljnssns"], + "codeowners": [ + "@domanchi", + "@gieljnssns" + ], "requirements": [], "documentation": "https://github.com/orgs/music-assistant/discussions/1160", "multi_instance": true diff --git a/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py b/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py index 9ba86ac8..00f450ce 100644 --- a/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py +++ b/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py @@ -131,30 +131,15 @@ class SoundcloudAsyncAPI: # the limit. However, we still implement pagination for future proofing. query_limit = 100 - url = ( - f"{BASE_URL}/me/track_likes/ids?client_id={self.client_id}&" - f"limit={query_limit}&app_version={self.app_version}" - ) num_items = 0 - while True: - response = await self.get(url, headers=self.headers) - - # Sanity check. - if "collection" not in response: - raise RuntimeError("Unexpected Soundcloud API response") - - for item in response["collection"]: - num_items += 1 - if limit > 0 and num_items >= limit: - return - - yield item - - # Handle case when results requested exceeds number of actual results. - if len(response["collection"]) < query_limit: + async for track in self._paginated_query( + "/me/track_likes/ids", params={"limit": str(query_limit)} + ): + num_items += 1 + if limit > 0 and num_items >= limit: return - url = response["next_href"] + yield track async def get_track_by_genre_recent(self, genre, limit=10): """Get track by genre recent. @@ -209,11 +194,10 @@ class SoundcloudAsyncAPI: # ---------------- PLAYLISTS ---------------- async def get_account_playlists(self): - """Get account playlists.""" - return await self.get( - f"{BASE_URL}/me/library/all?client_id{self.client_id}", - headers=self.headers, - ) + """Get account playlists, albums and stations.""" + # NOTE: This returns all track lists in reverse chronological order (most recent first). + async for playlist in self._paginated_query("/me/library/all"): + yield playlist async def get_playlist_details(self, playlist_id): """:param playlist_id: playlist id""" @@ -335,3 +319,44 @@ class SoundcloudAsyncAPI: f"&limit={limit}&offset=0&linked_partitioning=1&app_version={self.app_version}", headers=self.headers, ) + + async def _paginated_query( + self, + path: str, + params: dict[str, str] | None = None, + ) -> AsyncGenerator[list[dict[str, any]], None]: + """Paginate response queries. + + Soundcloud paginates its queries using the same pattern. As such, we leverage the + same pattern to implement a pagination pattern to iterate over their APIs. + + :param path: endpoint to query + :param params: key-value pairs to use as query parameters when constructing the initial URL + """ + if params is None: + params = {} + + url = f"{BASE_URL}{path}?client_id={self.client_id}&app_version={self.app_version}" + for k, v in params.values(): + url += f"&{k}={v}" + + while True: + response = await self.get(url, headers=self.headers) + + # Sanity check. + if "collection" not in response: + raise RuntimeError("Unexpected Soundcloud API response") + + for item in response["collection"]: + yield item + + # Handle case when results requested exceeds number of actual results. + if int(params.get("limit", 0)) and len(response["collection"]) < int(params["limit"]): + return + + try: + url = response["next_href"] + if not url: + return + except KeyError: + return -- 2.34.1