From e2dd07a413970a42e4aa304dd0c3cefbe5a1c0b7 Mon Sep 17 00:00:00 2001
From: Marcel van der Veldt <m.vanderveldt@outlook.com>
Date: Thu, 17 Oct 2024 12:43:10 +0200
Subject: [PATCH] Fix encoding detection

small tweak to encoding detection logic for (m3u) playlists
---
 music_assistant/server/helpers/audio.py     | 12 +++++++-----
 music_assistant/server/helpers/playlists.py |  9 ++++++---
 music_assistant/server/helpers/util.py      | 12 ++++++++++++
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/music_assistant/server/helpers/audio.py b/music_assistant/server/helpers/audio.py
index eda1fbb4..30a05ecb 100644
--- a/music_assistant/server/helpers/audio.py
+++ b/music_assistant/server/helpers/audio.py
@@ -46,7 +46,7 @@ from .playlists import HLS_CONTENT_TYPES, IsHLSPlaylist, PlaylistItem, fetch_pla
 from .process import AsyncProcess, check_output, communicate
 from .tags import parse_tags
 from .throttle_retry import BYPASS_THROTTLER
-from .util import TimedAsyncGenerator, create_tempfile
+from .util import TimedAsyncGenerator, create_tempfile, detect_charset
 
 if TYPE_CHECKING:
     from music_assistant.common.models.player_queue import QueueItem
@@ -603,8 +603,9 @@ async def get_hls_radio_stream(
             substream_url, headers=HTTP_HEADERS, timeout=timeout
         ) as resp:
             resp.raise_for_status()
-            charset = resp.charset or "utf-8"
-            substream_m3u_data = await resp.text(charset)
+            raw_data = await resp.read()
+            encoding = resp.charset or await detect_charset(raw_data)
+            substream_m3u_data = raw_data.decode(encoding)
         # get chunk-parts from the substream
         hls_chunks = parse_m3u(substream_m3u_data)
         chunk_seconds = 0
@@ -679,8 +680,9 @@ async def get_hls_substream(
         url, allow_redirects=True, headers=HTTP_HEADERS, timeout=timeout
     ) as resp:
         resp.raise_for_status()
-        charset = resp.charset or "utf-8"
-        master_m3u_data = await resp.text(charset)
+        raw_data = await resp.read()
+        encoding = resp.charset or await detect_charset(raw_data)
+        master_m3u_data = raw_data.decode(encoding)
     substreams = parse_m3u(master_m3u_data)
     if any(x for x in substreams if x.length and not x.key):
         # this is already a substream!
diff --git a/music_assistant/server/helpers/playlists.py b/music_assistant/server/helpers/playlists.py
index 8986de7f..66a159bc 100644
--- a/music_assistant/server/helpers/playlists.py
+++ b/music_assistant/server/helpers/playlists.py
@@ -11,6 +11,7 @@ from urllib.parse import urlparse
 from aiohttp import client_exceptions
 
 from music_assistant.common.models.errors import InvalidDataError
+from music_assistant.server.helpers.util import detect_charset
 
 if TYPE_CHECKING:
     from music_assistant.server import MusicAssistant
@@ -146,10 +147,12 @@ async def fetch_playlist(mass: MusicAssistant, url: str) -> list[PlaylistItem]:
     """Parse an online m3u or pls playlist."""
     try:
         async with mass.http_session.get(url, allow_redirects=True, timeout=5) as resp:
-            charset = resp.charset or "utf-8"
             try:
-                playlist_data = (await resp.content.read(64 * 1024)).decode(charset)
-            except ValueError as err:
+                raw_data = await resp.content.read(64 * 1024)
+                # NOTE: using resp.charset is not reliable, we need to detect it ourselves
+                encoding = resp.charset or await detect_charset(raw_data)
+                playlist_data = raw_data.decode(encoding, errors="replace")
+            except (ValueError, UnicodeDecodeError) as err:
                 msg = f"Could not decode playlist {url}"
                 raise InvalidDataError(msg) from err
     except TimeoutError as err:
diff --git a/music_assistant/server/helpers/util.py b/music_assistant/server/helpers/util.py
index 95896f6d..34ef1610 100644
--- a/music_assistant/server/helpers/util.py
+++ b/music_assistant/server/helpers/util.py
@@ -19,6 +19,7 @@ from importlib.metadata import version as pkg_version
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, ParamSpec, Self, TypeVar
 
+import cchardet as chardet
 import ifaddr
 import memory_tempfile
 from zeroconf import IPVersion
@@ -181,6 +182,17 @@ async def close_async_generator(agen: AsyncGenerator[Any, None]) -> None:
     await agen.aclose()
 
 
+async def detect_charset(data: bytes, fallback="utf-8") -> str:
+    """Detect charset of raw data."""
+    try:
+        detected = await asyncio.to_thread(chardet.detect, data)
+        if detected and detected["encoding"] and detected["confidence"] > 0.75:
+            return detected["encoding"]
+    except Exception as err:
+        LOGGER.debug("Failed to detect charset: %s", err)
+    return fallback
+
+
 class TaskManager:
     """
     Helper class to run many tasks at once.
-- 
2.34.1