ditch orjson for ujson
authorMarcel van der Veldt <m.vanderveldt@outlook.com>
Mon, 2 Nov 2020 23:26:03 +0000 (00:26 +0100)
committerMarcel van der Veldt <m.vanderveldt@outlook.com>
Mon, 2 Nov 2020 23:26:03 +0000 (00:26 +0100)
20 files changed:
Dockerfile
music_assistant/constants.py
music_assistant/helpers/musicbrainz.py
music_assistant/helpers/util.py
music_assistant/helpers/web.py
music_assistant/managers/config.py
music_assistant/models/config_entry.py
music_assistant/models/media_types.py
music_assistant/models/player.py
music_assistant/models/player_queue.py
music_assistant/models/player_state.py
music_assistant/models/streamdetails.py
music_assistant/providers/fanarttv/__init__.py
music_assistant/providers/spotify/__init__.py
music_assistant/web/endpoints/config.py
music_assistant/web/endpoints/login.py
music_assistant/web/endpoints/players.py
music_assistant/web/endpoints/playlists.py
music_assistant/web/endpoints/websocket.py
requirements.txt

index 852f8310d87da61bdf487cd544320af944b96129..d9925207d3bd80822760e9478cd77a1f8bd3a31e 100755 (executable)
@@ -1,4 +1,4 @@
-FROM python:3.8-alpine3.12
+FROM python:3.8-slim
 
 # Versions
 ARG JEMALLOC_VERSION=5.2.1
@@ -9,72 +9,30 @@ WORKDIR /tmp
 
 # Install packages
 RUN set -x \
-    && apk update \
-    && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-    # default packages
-    && apk add --no-cache \
-        tzdata \
-        ca-certificates \
-        curl \
-        bind-tools \
-        flac \
-        sox \
-        ffmpeg \
-        libsndfile \
-        taglib \
-        openblas \
-        libgfortran \
-        lapack \
-    # build packages
-    && apk add --no-cache --virtual .build-deps \
-        build-base \
-        libsndfile-dev \
-        taglib-dev \
-        openblas-dev \
-        lapack-dev \
-        libffi-dev \
-        gcc \
-        gfortran \
-        freetype-dev \
-        libpng-dev \
-        libressl-dev \
-        fribidi-dev \
-        harfbuzz-dev \
-        jpeg-dev \
-        lcms2-dev \
-        openjpeg-dev \
-        tcl-dev \
-        tiff-dev \
-        tk-dev \
-        zlib-dev
+    && apt-get update && apt-get install -y --no-install-recommends \
+        # required packages
+                   git jq tzdata curl ca-certificates flac sox libsox-fmt-all zip curl ffmpeg libsndfile1 libtag1v5 \
+        # build packages
+        # build-essential libtag1-dev libffi-dev\
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /usr/share/man/man1
 
 # setup jmalloc
-RUN curl -L -f -s "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" \
-        | tar -xjf - -C /usr/src \
+RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 | tar -xjf - -C /usr/src \
     && cd /usr/src/jemalloc-${JEMALLOC_VERSION} \
     && ./configure \
     && make \
     && make install \
     && rm -rf /usr/src/jemalloc-${JEMALLOC_VERSION} \
-    # change workdir back to /tmp
+    \
     && cd /tmp
 
-# dependencies for orjson
-ENV RUSTFLAGS "-C target-feature=-crt-static"
-RUN wget -O rustup.sh https://sh.rustup.rs \
-    && sh rustup.sh -y \
-    && cp $HOME/.cargo/bin/* /usr/local/bin \
-    && rustup install nightly \
-    && rustup default nightly
-
 # install uvloop and music assistant
 RUN pip install --upgrade uvloop music-assistant==${MASS_VERSION}
 
 # cleanup build files
-RUN rustup self uninstall -y \
-    && rm rustup.sh \
-    && apk del .build-deps \
-    && rm -rf /usr/src/*
+RUN apt-get purge -y --auto-remove libtag1-dev libffi-dev build-essential \
+    && rm -rf /var/lib/apt/lists/*
 
 
 ENV DEBUG=false
index 94a2018bbabbf9b9bd8303a560c08cb064c8fd6a..17a48b99024debe40480ff523da6584d1752d26b 100755 (executable)
@@ -1,6 +1,6 @@
 """All constants for Music Assistant."""
 
-__version__ = "0.0.61"
+__version__ = "0.0.62"
 REQUIRED_PYTHON_VER = "3.8"
 
 # configuration keys/attributes
index d4d66dd1af2bde1b6735ed044d3b1c1e6c8f1077..bebcbf8496e74d49a19b315acd38c86f0582046f 100644 (file)
@@ -2,10 +2,10 @@
 
 import logging
 import re
+from json.decoder import JSONDecodeError
 from typing import Optional
 
 import aiohttp
-import orjson
 from asyncio_throttle import Throttler
 from music_assistant.helpers.cache import async_use_cache
 from music_assistant.helpers.util import compare_strings, get_compare_string
@@ -172,10 +172,10 @@ class MusicBrainz:
                 url, headers=headers, params=params, verify_ssl=False
             ) as response:
                 try:
-                    result = await response.json(loads=orjson.loads)
+                    result = await response.json()
                 except (
                     aiohttp.client_exceptions.ContentTypeError,
-                    orjson.JSONDecodeError,
+                    JSONDecodeError,
                 ) as exc:
                     msg = await response.text()
                     LOGGER.error("%s - %s", str(exc), msg)
index 746ca49b239e456f4df44fbe2598f542c83c5c9a..3c5e9d72522c2c1c3be2537d3bd6c681a09827a3 100755 (executable)
@@ -1,6 +1,5 @@
 """Helper and utility functions."""
 import asyncio
-import functools
 import logging
 import os
 import platform
@@ -9,12 +8,13 @@ import socket
 import struct
 import tempfile
 import urllib.request
+from datetime import datetime
 from enum import Enum
 from io import BytesIO
 from typing import Any, Callable, TypeVar
 
 import memory_tempfile
-import orjson
+import ujson
 import unidecode
 
 # pylint: disable=invalid-name
@@ -236,9 +236,26 @@ def get_folder_size(folderpath):
     return total_size_gb
 
 
-# pylint: disable=invalid-name
-json_serializer = functools.partial(orjson.dumps, option=orjson.OPT_NAIVE_UTC)
-# pylint: enable=invalid-name
+def serialize_values(obj):
+    """Recursively create serializable values for (custom) data types."""
+
+    def get_val(val):
+        if hasattr(val, "to_dict"):
+            return val.to_dict()
+        if isinstance(val, list):
+            return [get_val(x) for x in val]
+        if isinstance(val, datetime):
+            return val.isoformat()
+        if isinstance(val, dict):
+            return {key: get_val(value) for key, value in val.items()}
+        return val
+
+    return get_val(obj)
+
+
+def json_serializer(obj):
+    """Json serializer to recursively create serializable values for custom data types."""
+    return ujson.dumps(serialize_values(obj))
 
 
 def get_compare_string(input_str):
@@ -268,9 +285,9 @@ def merge_dict(base_dict: dict, new_dict: dict):
 def try_load_json_file(jsonfile):
     """Try to load json from file."""
     try:
-        with open(jsonfile, "rb") as _file:
-            return orjson.loads(_file.read())
-    except (FileNotFoundError, orjson.JSONDecodeError) as exc:
+        with open(jsonfile, "r") as _file:
+            return ujson.loads(_file.read())
+    except (FileNotFoundError, ValueError) as exc:
         logging.getLogger().debug(
             "Could not load json from file %s", jsonfile, exc_info=exc
         )
index 207d9301034c7dfcb867c98447d3d60bcff0fab8..acdfff524bd53317e951214ee7dc218786ce779c 100644 (file)
@@ -22,9 +22,9 @@ async def async_stream_json(request: web.Request, generator: AsyncGenerator):
     async for item in generator:
         # write each item into the items object of the json
         if count:
-            json_response = b"," + json_serializer(item)
+            json_response = b"," + json_serializer(item).encode()
         else:
-            json_response = json_serializer(item)
+            json_response = json_serializer(item).encode()
         await resp.write(json_response)
         count += 1
     # write json close tag
index f2b54e6b87c006bfa6625a08c6ff21c6452c5cd1..1b43580614148b4d7d5d3c638b0bcd6b2b1ca5af 100755 (executable)
@@ -1,12 +1,12 @@
 """All classes and helpers for the Configuration."""
 
 import copy
+import json
 import logging
 import os
 import shutil
 from typing import List
 
-import orjson
 from music_assistant.constants import (
     CONF_CROSSFADE_DURATION,
     CONF_ENABLED,
@@ -283,8 +283,8 @@ class ConfigManager:
         if os.path.isfile(conf_file):
             shutil.move(conf_file, conf_file_backup)
         # write current config to file
-        with open(conf_file, "wb") as _file:
-            _file.write(orjson.dumps(self._stored_config, option=orjson.OPT_INDENT_2))
+        with open(conf_file, "w") as _file:
+            _file.write(json.dumps(self._stored_config, indent=4))
         LOGGER.info("Config saved!")
         self.loading = False
 
index 5dc8f989023c7d903826c398b7e9e88adc5ba292..0ff6850b3247360d139c1fa7340db49c1dae58cf 100644 (file)
@@ -4,6 +4,8 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, List, Tuple
 
+from mashumaro import DataClassDictMixin
+
 
 class ConfigEntryType(Enum):
     """Enum for the type of a config entry."""
@@ -17,7 +19,7 @@ class ConfigEntryType(Enum):
 
 
 @dataclass
-class ConfigEntry:
+class ConfigEntry(DataClassDictMixin):
     """Model for a Config Entry."""
 
     entry_key: str
index ca557bbf81754bdd1d97c46f85a088f5540aa5a3..6738f4a0edb26b85ece14a5bac7c3534053caadf 100755 (executable)
@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, List
 
+from mashumaro import DataClassDictMixin
 from music_assistant.helpers.util import CustomIntEnum
 
 
@@ -48,7 +49,7 @@ class TrackQuality(CustomIntEnum):
 
 
 @dataclass
-class MediaItemProviderId:
+class MediaItemProviderId(DataClassDictMixin):
     """Model for a MediaItem's provider id."""
 
     provider: str
@@ -66,7 +67,7 @@ class ExternalId(Enum):
 
 
 @dataclass
-class MediaItem:
+class MediaItem(DataClassDictMixin):
     """Representation of a media item."""
 
     item_id: str = ""
@@ -133,7 +134,7 @@ class Radio(MediaItem):
 
 
 @dataclass
-class SearchResult:
+class SearchResult(DataClassDictMixin):
     """Model for Media Item Search result."""
 
     artists: List[Artist] = field(default_factory=list)
index 8de28b83c005403e8495e52886871cfde3ba00cf..37450e764161c41dfe23c752b3810ac7516d64ca 100755 (executable)
@@ -5,6 +5,7 @@ from dataclasses import dataclass
 from enum import Enum
 from typing import Any, List, Optional
 
+from mashumaro import DataClassDictMixin
 from music_assistant.constants import EVENT_SET_PLAYER_CONTROL_STATE
 from music_assistant.helpers.typing import MusicAssistantType, QueueItems
 from music_assistant.helpers.util import CustomIntEnum, callback
@@ -21,7 +22,7 @@ class PlaybackState(Enum):
 
 
 @dataclass
-class DeviceInfo:
+class DeviceInfo(DataClassDictMixin):
     """Model for a player's deviceinfo."""
 
     model: str = ""
index fa26cb2ba573d7db8b450daf25624ce92be416a7..3924f06dff6d124d6e99e28823e1b843caf87865 100755 (executable)
@@ -541,9 +541,9 @@ class PlayerQueue:
             "cur_item_id": self.cur_item_id,
             "cur_index": self.cur_index,
             "next_index": self.next_index,
-            "cur_item": self.cur_item,
+            "cur_item": self.cur_item.to_dict() if self.cur_item else None,
             "cur_item_time": self.cur_item_time,
-            "next_item": self.next_item,
+            "next_item": self.next_item.to_dict() if self.next_item else None,
             "queue_stream_enabled": self.use_queue_stream,
         }
 
index 50414c7abbb59d439f168c9011b464e2c0d7874e..8e9b749b2f03eef984fb350d030564129f2a2bf3 100755 (executable)
@@ -399,8 +399,8 @@ class PlayerState:
             ATTR_MUTED: self.muted,
             ATTR_IS_GROUP_PLAYER: self.is_group_player,
             ATTR_GROUP_CHILDS: self.group_childs,
-            ATTR_DEVICE_INFO: self.device_info,
-            ATTR_UPDATED_AT: self.updated_at,
+            ATTR_DEVICE_INFO: self.device_info.to_dict(),
+            ATTR_UPDATED_AT: self.updated_at.isoformat(),
             ATTR_GROUP_PARENTS: self.group_parents,
             ATTR_FEATURES: self.features,
             ATTR_ACTIVE_QUEUE: self.active_queue,
index e3313f9fe3d49b50740cd5979978841e20c5ae8f..5a5e3cbf5d045478b7eaa4655451e54daefbed22 100644 (file)
@@ -4,6 +4,8 @@ from dataclasses import dataclass
 from enum import Enum
 from typing import Any
 
+from mashumaro import DataClassDictMixin
+
 
 class StreamType(Enum):
     """Enum with stream types."""
@@ -26,7 +28,7 @@ class ContentType(Enum):
 
 
 @dataclass
-class StreamDetails:
+class StreamDetails(DataClassDictMixin):
     """Model for streamdetails."""
 
     type: StreamType
index 66bb605c7be3854f12076287b049d43541bd9a06..55c0a984fca4673b562f73088b30fd840c7b1475 100755 (executable)
@@ -1,10 +1,10 @@
 """FanartTv Metadata provider."""
 
 import logging
+from json.decoder import JSONDecodeError
 from typing import Dict, List
 
 import aiohttp
-import orjson
 from asyncio_throttle import Throttler
 from music_assistant.models.config_entry import ConfigEntry
 from music_assistant.models.provider import MetadataProvider
@@ -90,10 +90,10 @@ class FanartTvProvider(MetadataProvider):
                 url, params=params, verify_ssl=False
             ) as response:
                 try:
-                    result = await response.json(loads=orjson.loads)
+                    result = await response.json()
                 except (
                     aiohttp.client_exceptions.ContentTypeError,
-                    orjson.JSONDecodeError,
+                    JSONDecodeError,
                 ):
                     LOGGER.error("Failed to retrieve %s", endpoint)
                     text_result = await response.text()
index 39d225745f26128b51237fe5d8795d6fdbcfe7f7..daff2a82b2a71085a47721b8fca0058a27508946 100644 (file)
@@ -1,12 +1,13 @@
 """Spotify musicprovider support for MusicAssistant."""
 import asyncio
+import json
 import logging
 import os
 import platform
 import time
+from json.decoder import JSONDecodeError
 from typing import List, Optional
 
-import orjson
 from asyncio_throttle import Throttler
 from music_assistant.constants import CONF_PASSWORD, CONF_USERNAME
 from music_assistant.helpers.app_vars import get_app_var  # noqa # pylint: disable=all
@@ -527,8 +528,8 @@ class SpotifyProvider(MusicProvider):
         )
         stdout, _ = await spotty.communicate()
         try:
-            result = orjson.loads(stdout)
-        except orjson.JSONDecodeError:
+            result = json.loads(stdout)
+        except JSONDecodeError:
             LOGGER.warning("Error while retrieving Spotify token!")
             return None
         # transform token info to spotipy compatible format
@@ -569,7 +570,7 @@ class SpotifyProvider(MusicProvider):
             async with self.mass.http_session.get(
                 url, headers=headers, params=params, verify_ssl=False
             ) as response:
-                result = await response.json(loads=orjson.loads)
+                result = await response.json()
                 if not result or "error" in result:
                     LOGGER.error("%s - %s", endpoint, result)
                     result = None
index 6933eb0d00a8b916ea6ea9da9df64423e983055c..04df56d353c83b072bd4d06a309fad9675d13a9d 100644 (file)
@@ -1,6 +1,7 @@
 """Config API endpoints."""
 
-import orjson
+from json.decoder import JSONDecodeError
+
 from aiohttp.web import Request, Response, RouteTableDef, json_response
 from aiohttp_jwt import login_required
 from music_assistant.constants import (
@@ -63,8 +64,8 @@ async def async_put_config(request: Request):
     conf_base = request.match_info.get("base")
     entry_key = request.match_info.get("entry_key")
     try:
-        new_value = await request.json(loads=orjson.loads)
-    except orjson.JSONDecodeError:
+        new_value = await request.json()
+    except JSONDecodeError:
         new_value = (
             request.app["mass"]
             .config[conf_base][conf_key]
index 94fe92ad820d5c9c7641138ce826fad96d950a01..6dbe9ee62367d618f96fbc31ded78a6f90fb378c 100644 (file)
@@ -40,7 +40,7 @@ async def async_get_token(
         return {
             "user": username,
             "token": token.decode(),
-            "expires": token_expires,
+            "expires": token_expires.isoformat(),
             "scopes": scopes,
         }
     return None
index bac9e3b307be245f6ce7731a0c7a6d677ab77a30..c40e018b03fa83e43dadbbf3af10fd95a950b4bb 100644 (file)
@@ -1,7 +1,8 @@
 """Players API endpoints."""
 
-import orjson
-from aiohttp.web import Request, Response, RouteTableDef
+from json.decoder import JSONDecodeError
+
+from aiohttp.web import Request, Response, RouteTableDef, json_response
 from aiohttp_jwt import login_required
 from music_assistant.helpers.util import json_serializer
 from music_assistant.helpers.web import async_media_items_from_body, async_stream_json
@@ -29,10 +30,10 @@ async def async_player_command(request: Request):
     player_id = request.match_info.get("player_id")
     cmd = request.match_info.get("cmd")
     try:
-        cmd_args = await request.json(loads=orjson.loads)
+        cmd_args = await request.json()
         if cmd_args in ["", {}, []]:
             cmd_args = None
-    except orjson.JSONDecodeError:
+    except JSONDecodeError:
         cmd_args = None
     player_cmd = getattr(request.app["mass"].players, f"async_cmd_{cmd}", None)
     if player_cmd and cmd_args is not None:
@@ -60,7 +61,7 @@ async def async_player_play_media(request: Request):
         player_id, media_items, queue_opt
     )
     result = {"success": success in [True, None]}
-    return Response(body=json_serializer(result), content_type="application/json")
+    return json_response(result)
 
 
 @routes.get("/api/players/{player_id}/queue/items/{queue_item}")
@@ -77,7 +78,7 @@ async def async_player_queue_item(request: Request):
         queue_item = player_queue.get_item(item_id)
     except ValueError:
         queue_item = player_queue.by_item_id(item_id)
-    return Response(body=json_serializer(queue_item), content_type="application/json")
+    return json_response(queue_item.to_dict())
 
 
 @routes.get("/api/players/{player_id}/queue/items")
@@ -117,8 +118,8 @@ async def async_player_queue_cmd(request: Request):
     player_queue = request.app["mass"].players.get_player_queue(player_id)
     cmd = request.match_info.get("cmd")
     try:
-        cmd_args = await request.json(loads=orjson.loads)
-    except orjson.JSONDecodeError:
+        cmd_args = await request.json()
+    except JSONDecodeError:
         cmd_args = None
     if cmd == "repeat_enabled":
         player_queue.repeat_enabled = cmd_args
index 080e08c0c0b1f08692a65285d738f0595e527252..b5830ba85b428a6539abd8dfcc19a87668c7e0f9 100644 (file)
@@ -1,8 +1,8 @@
 """Playlists API endpoints."""
 
-from aiohttp.web import Request, Response, RouteTableDef
+import ujson
+from aiohttp.web import Request, Response, RouteTableDef, json_response
 from aiohttp_jwt import login_required
-from music_assistant.helpers.util import json_serializer
 from music_assistant.helpers.web import async_media_items_from_body, async_stream_json
 
 routes = RouteTableDef()
@@ -17,7 +17,7 @@ async def async_playlist(request: Request):
     if item_id is None or provider is None:
         return Response(text="invalid item or provider", status=501)
     result = await request.app["mass"].music.async_get_playlist(item_id, provider)
-    return Response(body=json_serializer(result), content_type="application/json")
+    return json_response(result)
 
 
 @routes.get("/api/playlists/{item_id}/tracks")
@@ -37,10 +37,10 @@ async def async_playlist_tracks(request: Request):
 async def async_add_playlist_tracks(request: Request):
     """Add tracks to (editable) playlist."""
     item_id = request.match_info.get("item_id")
-    body = await request.json()
+    body = await request.json(loads=ujson.loads)
     tracks = await async_media_items_from_body(request.app["mass"], body)
     result = await request.app["mass"].music.async_add_playlist_tracks(item_id, tracks)
-    return Response(body=json_serializer(result), content_type="application/json")
+    return json_response(result)
 
 
 @routes.delete("/api/playlists/{item_id}/tracks")
@@ -48,9 +48,9 @@ async def async_add_playlist_tracks(request: Request):
 async def async_remove_playlist_tracks(request: Request):
     """Remove tracks from (editable) playlist."""
     item_id = request.match_info.get("item_id")
-    body = await request.json()
+    body = await request.json(loads=ujson.loads)
     tracks = await async_media_items_from_body(request.app["mass"], body)
     result = await request.app["mass"].music.async_remove_playlist_tracks(
         item_id, tracks
     )
-    return Response(body=json_serializer(result), content_type="application/json")
+    return json_response(result)
index 13cbf4862306246126cbce07986a18a0024bfd80..ca6894ab56ac67eff95cfb61b7da6e7746588bec 100644 (file)
@@ -3,7 +3,7 @@
 import logging
 
 import jwt
-import orjson
+import ujson
 from aiohttp import WSMsgType
 from aiohttp.web import Request, RouteTableDef, WebSocketResponse
 from music_assistant.helpers.util import json_serializer
@@ -29,7 +29,7 @@ async def async_websocket_handler(request: Request):
                 msg_details = msg_details.to_dict()
             ws_msg = {"message": msg, "message_details": msg_details}
             try:
-                await ws_response.send_str(json_serializer(ws_msg).decode())
+                await ws_response.send_str(json_serializer(ws_msg))
             # pylint: disable=broad-except
             except Exception as exc:
                 LOGGER.debug(
@@ -44,8 +44,8 @@ async def async_websocket_handler(request: Request):
                 LOGGER.warning(msg.data)
                 continue
             try:
-                data = msg.json(loads=orjson.loads)
-            except orjson.JSONDecodeError:
+                data = msg.json(loads=ujson.loads)
+            except ValueError:
                 await async_send_message(
                     "error",
                     'commands must be issued in json format \
index f1ee27220c8216d4f599291ce6a75a222d3eb5f3..78cbfcb812d72de0835f492d6e340efb69f20d0a 100644 (file)
@@ -22,4 +22,5 @@ aiohttp_jwt==0.6.1
 zeroconf==0.28.6
 passlib==1.7.4
 cryptography==3.2
-orjson==3.4.3
+ujson==4.0.1
+mashumaro==1.13