[soundcloud] Improved playlist iteration (#924)

* [soundcloud] improved playlist imports

Similar to https://github.com/music-assistant/server/pull/922, the
playlist import process is incomplete. Through trial and error, I
discovered that soundcloud returns all forms of "track lists" as a
response to the API call -- and that it was sorted by recency (which
explained the behavior I witnessed on `2023.6.0b6`.

This PR fixes it right up. It generalizes soundcloud pagination for
easier future adoption and for the soundcloud playlists.

Manually tested the `asyncsoundcloudpy` API.

* added self to manifest codeowners

diff --git a/music_assistant/server/providers/soundcloud/__init__.py b/music_assistant/server/providers/soundcloud/__init__.py
index 5941cd5b4714406ee979550d0427ca0ebf6eb6fe..3346752b6a59329b8c33f8a1cc5fa2eb20f65285 100644 (file)
--- a/music_assistant/server/providers/soundcloud/__init__.py
+++ b/music_assistant/server/providers/soundcloud/__init__.py
@@ -174,21 +174,33 @@ class SoundcloudMusicProvider(MusicProvider):
     async def get_library_playlists(self) -> AsyncGenerator[Playlist, None]:
         """Retrieve all library playlists from Soundcloud."""
         time_start = time.time()
-        playlists = await self._soundcloud.get_account_playlists()
-        self.logger.debug(
-            "Processing Soundcloud library playlists took %s seconds",
-            round(time.time() - time_start, 2),
-        )
-        for item in playlists["collection"]:
+        async for item in self._soundcloud.get_account_playlists():
+            try:
+                raw_playlist = item["playlist"]
+            except KeyError:
+                self.logger.debug(
+                    "Unexpected Soundcloud API response when parsing playlists: %s",
+                    item,
+                )
+                continue
+
             try:
-                playlist_obj = await self._soundcloud.get_playlist_details(
-                    playlist_id=item["playlist"]["id"]
+                playlist = await self._soundcloud.get_playlist_details(
+                    playlist_id=raw_playlist["id"],
                 )
-                yield await self._parse_playlist(playlist_obj)
+
+                yield await self._parse_playlist(playlist)
             except (KeyError, TypeError, InvalidDataError, IndexError) as error:
-                self.logger.debug("Parse playlist failed: %s", playlist_obj, exc_info=error)
+                self.logger.debug(
+                    "Failed to obtain Soundcloud playlist details: %s", raw_playlist, exc_info=error
+                )
                 continue
 
+        self.logger.debug(
+            "Processing Soundcloud library playlists took %s seconds",
+            round(time.time() - time_start, 2),
+        )
+
     async def get_library_tracks(self) -> AsyncGenerator[Track, None]:
         """Retrieve library tracks from Soundcloud."""
         time_start = time.time()

diff --git a/music_assistant/server/providers/soundcloud/manifest.json b/music_assistant/server/providers/soundcloud/manifest.json
index b56ebd1c6bd91759e056fcb14160cfc6ee8bdb44..4d4378ae10360c4e567acfcfce85bcdbe5de9b38 100644 (file)
--- a/music_assistant/server/providers/soundcloud/manifest.json
+++ b/music_assistant/server/providers/soundcloud/manifest.json
@@ -3,7 +3,10 @@
   "domain": "soundcloud",
   "name": "Soundcloud",
   "description": "Support for the Soundcloud streaming provider in Music Assistant.",
-  "codeowners": ["@gieljnssns"],
+  "codeowners": [
+    "@domanchi",
+    "@gieljnssns"
+  ],
   "requirements": [],
   "documentation": "https://github.com/orgs/music-assistant/discussions/1160",
   "multi_instance": true

diff --git a/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py b/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py
index 9ba86ac8a140ea518728419b9486f49aa2d17d1c..00f450ce03be74ea1f84a70761815595863b77bd 100644 (file)
--- a/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py
+++ b/music_assistant/server/providers/soundcloud/soundcloudpy/asyncsoundcloudpy.py
@@ -131,30 +131,15 @@ class SoundcloudAsyncAPI:
             # the limit. However, we still implement pagination for future proofing.
             query_limit = 100
 
-        url = (
-            f"{BASE_URL}/me/track_likes/ids?client_id={self.client_id}&"
-            f"limit={query_limit}&app_version={self.app_version}"
-        )
         num_items = 0
-        while True:
-            response = await self.get(url, headers=self.headers)
-
-            # Sanity check.
-            if "collection" not in response:
-                raise RuntimeError("Unexpected Soundcloud API response")
-
-            for item in response["collection"]:
-                num_items += 1
-                if limit > 0 and num_items >= limit:
-                    return
-
-                yield item
-
-            # Handle case when results requested exceeds number of actual results.
-            if len(response["collection"]) < query_limit:
+        async for track in self._paginated_query(
+            "/me/track_likes/ids", params={"limit": str(query_limit)}
+        ):
+            num_items += 1
+            if limit > 0 and num_items >= limit:
                 return
 
-            url = response["next_href"]
+            yield track
 
     async def get_track_by_genre_recent(self, genre, limit=10):
         """Get track by genre recent.
@@ -209,11 +194,10 @@ class SoundcloudAsyncAPI:
     # ---------------- PLAYLISTS ----------------
 
     async def get_account_playlists(self):
-        """Get account playlists."""
-        return await self.get(
-            f"{BASE_URL}/me/library/all?client_id{self.client_id}",
-            headers=self.headers,
-        )
+        """Get account playlists, albums and stations."""
+        # NOTE: This returns all track lists in reverse chronological order (most recent first).
+        async for playlist in self._paginated_query("/me/library/all"):
+            yield playlist
 
     async def get_playlist_details(self, playlist_id):
         """:param playlist_id: playlist id"""
@@ -335,3 +319,44 @@ class SoundcloudAsyncAPI:
             f"&limit={limit}&offset=0&linked_partitioning=1&app_version={self.app_version}",
             headers=self.headers,
         )
+
+    async def _paginated_query(
+        self,
+        path: str,
+        params: dict[str, str] | None = None,
+    ) -> AsyncGenerator[list[dict[str, any]], None]:
+        """Paginate response queries.
+
+        Soundcloud paginates its queries using the same pattern. As such, we leverage the
+        same pattern to implement a pagination pattern to iterate over their APIs.
+
+        :param path: endpoint to query
+        :param params: key-value pairs to use as query parameters when constructing the initial URL
+        """
+        if params is None:
+            params = {}
+
+        url = f"{BASE_URL}{path}?client_id={self.client_id}&app_version={self.app_version}"
+        for k, v in params.values():
+            url += f"&{k}={v}"
+
+        while True:
+            response = await self.get(url, headers=self.headers)
+
+            # Sanity check.
+            if "collection" not in response:
+                raise RuntimeError("Unexpected Soundcloud API response")
+
+            for item in response["collection"]:
+                yield item
+
+            # Handle case when results requested exceeds number of actual results.
+            if int(params.get("limit", 0)) and len(response["collection"]) < int(params["limit"]):
+                return
+
+            try:
+                url = response["next_href"]
+                if not url:
+                    return
+            except KeyError:
+                return