From 2e618e2b328002b0b86a2243cf928d33ca7c9721 Mon Sep 17 00:00:00 2001 From: Marcel van der Veldt Date: Tue, 3 Nov 2020 00:26:03 +0100 Subject: [PATCH] ditch orjson for ujson --- Dockerfile | 66 ++++--------------- music_assistant/constants.py | 2 +- music_assistant/helpers/musicbrainz.py | 6 +- music_assistant/helpers/util.py | 33 +++++++--- music_assistant/helpers/web.py | 4 +- music_assistant/managers/config.py | 6 +- music_assistant/models/config_entry.py | 4 +- music_assistant/models/media_types.py | 7 +- music_assistant/models/player.py | 3 +- music_assistant/models/player_queue.py | 4 +- music_assistant/models/player_state.py | 4 +- music_assistant/models/streamdetails.py | 4 +- .../providers/fanarttv/__init__.py | 6 +- music_assistant/providers/spotify/__init__.py | 9 +-- music_assistant/web/endpoints/config.py | 7 +- music_assistant/web/endpoints/login.py | 2 +- music_assistant/web/endpoints/players.py | 17 ++--- music_assistant/web/endpoints/playlists.py | 14 ++-- music_assistant/web/endpoints/websocket.py | 8 +-- requirements.txt | 3 +- 20 files changed, 97 insertions(+), 112 deletions(-) diff --git a/Dockerfile b/Dockerfile index 852f8310..d9925207 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-alpine3.12 +FROM python:3.8-slim # Versions ARG JEMALLOC_VERSION=5.2.1 @@ -9,72 +9,30 @@ WORKDIR /tmp # Install packages RUN set -x \ - && apk update \ - && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ - # default packages - && apk add --no-cache \ - tzdata \ - ca-certificates \ - curl \ - bind-tools \ - flac \ - sox \ - ffmpeg \ - libsndfile \ - taglib \ - openblas \ - libgfortran \ - lapack \ - # build packages - && apk add --no-cache --virtual .build-deps \ - build-base \ - libsndfile-dev \ - taglib-dev \ - openblas-dev \ - lapack-dev \ - libffi-dev \ - gcc \ - gfortran \ - freetype-dev \ - libpng-dev \ - libressl-dev \ - fribidi-dev \ - harfbuzz-dev \ - jpeg-dev \ - lcms2-dev \ - openjpeg-dev \ - tcl-dev \ - tiff-dev \ - tk-dev \ - zlib-dev + && apt-get update && apt-get install -y --no-install-recommends \ + # required packages + git jq tzdata curl ca-certificates flac sox libsox-fmt-all zip curl ffmpeg libsndfile1 libtag1v5 \ + # build packages + # build-essential libtag1-dev libffi-dev\ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /usr/share/man/man1 # setup jmalloc -RUN curl -L -f -s "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" \ - | tar -xjf - -C /usr/src \ +RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 | tar -xjf - -C /usr/src \ && cd /usr/src/jemalloc-${JEMALLOC_VERSION} \ && ./configure \ && make \ && make install \ && rm -rf /usr/src/jemalloc-${JEMALLOC_VERSION} \ - # change workdir back to /tmp + \ && cd /tmp -# dependencies for orjson -ENV RUSTFLAGS "-C target-feature=-crt-static" -RUN wget -O rustup.sh https://sh.rustup.rs \ - && sh rustup.sh -y \ - && cp $HOME/.cargo/bin/* /usr/local/bin \ - && rustup install nightly \ - && rustup default nightly - # install uvloop and music assistant RUN pip install --upgrade uvloop music-assistant==${MASS_VERSION} # cleanup build files -RUN rustup self uninstall -y \ - && rm rustup.sh \ - && apk del .build-deps \ - && rm -rf /usr/src/* +RUN apt-get purge -y --auto-remove libtag1-dev libffi-dev build-essential \ + && rm -rf /var/lib/apt/lists/* ENV DEBUG=false diff --git a/music_assistant/constants.py b/music_assistant/constants.py index 94a2018b..17a48b99 100755 --- a/music_assistant/constants.py +++ b/music_assistant/constants.py @@ -1,6 +1,6 @@ """All constants for Music Assistant.""" -__version__ = "0.0.61" +__version__ = "0.0.62" REQUIRED_PYTHON_VER = "3.8" # configuration keys/attributes diff --git a/music_assistant/helpers/musicbrainz.py b/music_assistant/helpers/musicbrainz.py index d4d66dd1..bebcbf84 100644 --- a/music_assistant/helpers/musicbrainz.py +++ b/music_assistant/helpers/musicbrainz.py @@ -2,10 +2,10 @@ import logging import re +from json.decoder import JSONDecodeError from typing import Optional import aiohttp -import orjson from asyncio_throttle import Throttler from music_assistant.helpers.cache import async_use_cache from music_assistant.helpers.util import compare_strings, get_compare_string @@ -172,10 +172,10 @@ class MusicBrainz: url, headers=headers, params=params, verify_ssl=False ) as response: try: - result = await response.json(loads=orjson.loads) + result = await response.json() except ( aiohttp.client_exceptions.ContentTypeError, - orjson.JSONDecodeError, + JSONDecodeError, ) as exc: msg = await response.text() LOGGER.error("%s - %s", str(exc), msg) diff --git a/music_assistant/helpers/util.py b/music_assistant/helpers/util.py index 746ca49b..3c5e9d72 100755 --- a/music_assistant/helpers/util.py +++ b/music_assistant/helpers/util.py @@ -1,6 +1,5 @@ """Helper and utility functions.""" import asyncio -import functools import logging import os import platform @@ -9,12 +8,13 @@ import socket import struct import tempfile import urllib.request +from datetime import datetime from enum import Enum from io import BytesIO from typing import Any, Callable, TypeVar import memory_tempfile -import orjson +import ujson import unidecode # pylint: disable=invalid-name @@ -236,9 +236,26 @@ def get_folder_size(folderpath): return total_size_gb -# pylint: disable=invalid-name -json_serializer = functools.partial(orjson.dumps, option=orjson.OPT_NAIVE_UTC) -# pylint: enable=invalid-name +def serialize_values(obj): + """Recursively create serializable values for (custom) data types.""" + + def get_val(val): + if hasattr(val, "to_dict"): + return val.to_dict() + if isinstance(val, list): + return [get_val(x) for x in val] + if isinstance(val, datetime): + return val.isoformat() + if isinstance(val, dict): + return {key: get_val(value) for key, value in val.items()} + return val + + return get_val(obj) + + +def json_serializer(obj): + """Json serializer to recursively create serializable values for custom data types.""" + return ujson.dumps(serialize_values(obj)) def get_compare_string(input_str): @@ -268,9 +285,9 @@ def merge_dict(base_dict: dict, new_dict: dict): def try_load_json_file(jsonfile): """Try to load json from file.""" try: - with open(jsonfile, "rb") as _file: - return orjson.loads(_file.read()) - except (FileNotFoundError, orjson.JSONDecodeError) as exc: + with open(jsonfile, "r") as _file: + return ujson.loads(_file.read()) + except (FileNotFoundError, ValueError) as exc: logging.getLogger().debug( "Could not load json from file %s", jsonfile, exc_info=exc ) diff --git a/music_assistant/helpers/web.py b/music_assistant/helpers/web.py index 207d9301..acdfff52 100644 --- a/music_assistant/helpers/web.py +++ b/music_assistant/helpers/web.py @@ -22,9 +22,9 @@ async def async_stream_json(request: web.Request, generator: AsyncGenerator): async for item in generator: # write each item into the items object of the json if count: - json_response = b"," + json_serializer(item) + json_response = b"," + json_serializer(item).encode() else: - json_response = json_serializer(item) + json_response = json_serializer(item).encode() await resp.write(json_response) count += 1 # write json close tag diff --git a/music_assistant/managers/config.py b/music_assistant/managers/config.py index f2b54e6b..1b435806 100755 --- a/music_assistant/managers/config.py +++ b/music_assistant/managers/config.py @@ -1,12 +1,12 @@ """All classes and helpers for the Configuration.""" import copy +import json import logging import os import shutil from typing import List -import orjson from music_assistant.constants import ( CONF_CROSSFADE_DURATION, CONF_ENABLED, @@ -283,8 +283,8 @@ class ConfigManager: if os.path.isfile(conf_file): shutil.move(conf_file, conf_file_backup) # write current config to file - with open(conf_file, "wb") as _file: - _file.write(orjson.dumps(self._stored_config, option=orjson.OPT_INDENT_2)) + with open(conf_file, "w") as _file: + _file.write(json.dumps(self._stored_config, indent=4)) LOGGER.info("Config saved!") self.loading = False diff --git a/music_assistant/models/config_entry.py b/music_assistant/models/config_entry.py index 5dc8f989..0ff6850b 100644 --- a/music_assistant/models/config_entry.py +++ b/music_assistant/models/config_entry.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field from enum import Enum from typing import Any, List, Tuple +from mashumaro import DataClassDictMixin + class ConfigEntryType(Enum): """Enum for the type of a config entry.""" @@ -17,7 +19,7 @@ class ConfigEntryType(Enum): @dataclass -class ConfigEntry: +class ConfigEntry(DataClassDictMixin): """Model for a Config Entry.""" entry_key: str diff --git a/music_assistant/models/media_types.py b/music_assistant/models/media_types.py index ca557bbf..6738f4a0 100755 --- a/music_assistant/models/media_types.py +++ b/music_assistant/models/media_types.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from enum import Enum from typing import Any, List +from mashumaro import DataClassDictMixin from music_assistant.helpers.util import CustomIntEnum @@ -48,7 +49,7 @@ class TrackQuality(CustomIntEnum): @dataclass -class MediaItemProviderId: +class MediaItemProviderId(DataClassDictMixin): """Model for a MediaItem's provider id.""" provider: str @@ -66,7 +67,7 @@ class ExternalId(Enum): @dataclass -class MediaItem: +class MediaItem(DataClassDictMixin): """Representation of a media item.""" item_id: str = "" @@ -133,7 +134,7 @@ class Radio(MediaItem): @dataclass -class SearchResult: +class SearchResult(DataClassDictMixin): """Model for Media Item Search result.""" artists: List[Artist] = field(default_factory=list) diff --git a/music_assistant/models/player.py b/music_assistant/models/player.py index 8de28b83..37450e76 100755 --- a/music_assistant/models/player.py +++ b/music_assistant/models/player.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from enum import Enum from typing import Any, List, Optional +from mashumaro import DataClassDictMixin from music_assistant.constants import EVENT_SET_PLAYER_CONTROL_STATE from music_assistant.helpers.typing import MusicAssistantType, QueueItems from music_assistant.helpers.util import CustomIntEnum, callback @@ -21,7 +22,7 @@ class PlaybackState(Enum): @dataclass -class DeviceInfo: +class DeviceInfo(DataClassDictMixin): """Model for a player's deviceinfo.""" model: str = "" diff --git a/music_assistant/models/player_queue.py b/music_assistant/models/player_queue.py index fa26cb2b..3924f06d 100755 --- a/music_assistant/models/player_queue.py +++ b/music_assistant/models/player_queue.py @@ -541,9 +541,9 @@ class PlayerQueue: "cur_item_id": self.cur_item_id, "cur_index": self.cur_index, "next_index": self.next_index, - "cur_item": self.cur_item, + "cur_item": self.cur_item.to_dict() if self.cur_item else None, "cur_item_time": self.cur_item_time, - "next_item": self.next_item, + "next_item": self.next_item.to_dict() if self.next_item else None, "queue_stream_enabled": self.use_queue_stream, } diff --git a/music_assistant/models/player_state.py b/music_assistant/models/player_state.py index 50414c7a..8e9b749b 100755 --- a/music_assistant/models/player_state.py +++ b/music_assistant/models/player_state.py @@ -399,8 +399,8 @@ class PlayerState: ATTR_MUTED: self.muted, ATTR_IS_GROUP_PLAYER: self.is_group_player, ATTR_GROUP_CHILDS: self.group_childs, - ATTR_DEVICE_INFO: self.device_info, - ATTR_UPDATED_AT: self.updated_at, + ATTR_DEVICE_INFO: self.device_info.to_dict(), + ATTR_UPDATED_AT: self.updated_at.isoformat(), ATTR_GROUP_PARENTS: self.group_parents, ATTR_FEATURES: self.features, ATTR_ACTIVE_QUEUE: self.active_queue, diff --git a/music_assistant/models/streamdetails.py b/music_assistant/models/streamdetails.py index e3313f9f..5a5e3cbf 100644 --- a/music_assistant/models/streamdetails.py +++ b/music_assistant/models/streamdetails.py @@ -4,6 +4,8 @@ from dataclasses import dataclass from enum import Enum from typing import Any +from mashumaro import DataClassDictMixin + class StreamType(Enum): """Enum with stream types.""" @@ -26,7 +28,7 @@ class ContentType(Enum): @dataclass -class StreamDetails: +class StreamDetails(DataClassDictMixin): """Model for streamdetails.""" type: StreamType diff --git a/music_assistant/providers/fanarttv/__init__.py b/music_assistant/providers/fanarttv/__init__.py index 66bb605c..55c0a984 100755 --- a/music_assistant/providers/fanarttv/__init__.py +++ b/music_assistant/providers/fanarttv/__init__.py @@ -1,10 +1,10 @@ """FanartTv Metadata provider.""" import logging +from json.decoder import JSONDecodeError from typing import Dict, List import aiohttp -import orjson from asyncio_throttle import Throttler from music_assistant.models.config_entry import ConfigEntry from music_assistant.models.provider import MetadataProvider @@ -90,10 +90,10 @@ class FanartTvProvider(MetadataProvider): url, params=params, verify_ssl=False ) as response: try: - result = await response.json(loads=orjson.loads) + result = await response.json() except ( aiohttp.client_exceptions.ContentTypeError, - orjson.JSONDecodeError, + JSONDecodeError, ): LOGGER.error("Failed to retrieve %s", endpoint) text_result = await response.text() diff --git a/music_assistant/providers/spotify/__init__.py b/music_assistant/providers/spotify/__init__.py index 39d22574..daff2a82 100644 --- a/music_assistant/providers/spotify/__init__.py +++ b/music_assistant/providers/spotify/__init__.py @@ -1,12 +1,13 @@ """Spotify musicprovider support for MusicAssistant.""" import asyncio +import json import logging import os import platform import time +from json.decoder import JSONDecodeError from typing import List, Optional -import orjson from asyncio_throttle import Throttler from music_assistant.constants import CONF_PASSWORD, CONF_USERNAME from music_assistant.helpers.app_vars import get_app_var # noqa # pylint: disable=all @@ -527,8 +528,8 @@ class SpotifyProvider(MusicProvider): ) stdout, _ = await spotty.communicate() try: - result = orjson.loads(stdout) - except orjson.JSONDecodeError: + result = json.loads(stdout) + except JSONDecodeError: LOGGER.warning("Error while retrieving Spotify token!") return None # transform token info to spotipy compatible format @@ -569,7 +570,7 @@ class SpotifyProvider(MusicProvider): async with self.mass.http_session.get( url, headers=headers, params=params, verify_ssl=False ) as response: - result = await response.json(loads=orjson.loads) + result = await response.json() if not result or "error" in result: LOGGER.error("%s - %s", endpoint, result) result = None diff --git a/music_assistant/web/endpoints/config.py b/music_assistant/web/endpoints/config.py index 6933eb0d..04df56d3 100644 --- a/music_assistant/web/endpoints/config.py +++ b/music_assistant/web/endpoints/config.py @@ -1,6 +1,7 @@ """Config API endpoints.""" -import orjson +from json.decoder import JSONDecodeError + from aiohttp.web import Request, Response, RouteTableDef, json_response from aiohttp_jwt import login_required from music_assistant.constants import ( @@ -63,8 +64,8 @@ async def async_put_config(request: Request): conf_base = request.match_info.get("base") entry_key = request.match_info.get("entry_key") try: - new_value = await request.json(loads=orjson.loads) - except orjson.JSONDecodeError: + new_value = await request.json() + except JSONDecodeError: new_value = ( request.app["mass"] .config[conf_base][conf_key] diff --git a/music_assistant/web/endpoints/login.py b/music_assistant/web/endpoints/login.py index 94fe92ad..6dbe9ee6 100644 --- a/music_assistant/web/endpoints/login.py +++ b/music_assistant/web/endpoints/login.py @@ -40,7 +40,7 @@ async def async_get_token( return { "user": username, "token": token.decode(), - "expires": token_expires, + "expires": token_expires.isoformat(), "scopes": scopes, } return None diff --git a/music_assistant/web/endpoints/players.py b/music_assistant/web/endpoints/players.py index bac9e3b3..c40e018b 100644 --- a/music_assistant/web/endpoints/players.py +++ b/music_assistant/web/endpoints/players.py @@ -1,7 +1,8 @@ """Players API endpoints.""" -import orjson -from aiohttp.web import Request, Response, RouteTableDef +from json.decoder import JSONDecodeError + +from aiohttp.web import Request, Response, RouteTableDef, json_response from aiohttp_jwt import login_required from music_assistant.helpers.util import json_serializer from music_assistant.helpers.web import async_media_items_from_body, async_stream_json @@ -29,10 +30,10 @@ async def async_player_command(request: Request): player_id = request.match_info.get("player_id") cmd = request.match_info.get("cmd") try: - cmd_args = await request.json(loads=orjson.loads) + cmd_args = await request.json() if cmd_args in ["", {}, []]: cmd_args = None - except orjson.JSONDecodeError: + except JSONDecodeError: cmd_args = None player_cmd = getattr(request.app["mass"].players, f"async_cmd_{cmd}", None) if player_cmd and cmd_args is not None: @@ -60,7 +61,7 @@ async def async_player_play_media(request: Request): player_id, media_items, queue_opt ) result = {"success": success in [True, None]} - return Response(body=json_serializer(result), content_type="application/json") + return json_response(result) @routes.get("/api/players/{player_id}/queue/items/{queue_item}") @@ -77,7 +78,7 @@ async def async_player_queue_item(request: Request): queue_item = player_queue.get_item(item_id) except ValueError: queue_item = player_queue.by_item_id(item_id) - return Response(body=json_serializer(queue_item), content_type="application/json") + return json_response(queue_item.to_dict()) @routes.get("/api/players/{player_id}/queue/items") @@ -117,8 +118,8 @@ async def async_player_queue_cmd(request: Request): player_queue = request.app["mass"].players.get_player_queue(player_id) cmd = request.match_info.get("cmd") try: - cmd_args = await request.json(loads=orjson.loads) - except orjson.JSONDecodeError: + cmd_args = await request.json() + except JSONDecodeError: cmd_args = None if cmd == "repeat_enabled": player_queue.repeat_enabled = cmd_args diff --git a/music_assistant/web/endpoints/playlists.py b/music_assistant/web/endpoints/playlists.py index 080e08c0..b5830ba8 100644 --- a/music_assistant/web/endpoints/playlists.py +++ b/music_assistant/web/endpoints/playlists.py @@ -1,8 +1,8 @@ """Playlists API endpoints.""" -from aiohttp.web import Request, Response, RouteTableDef +import ujson +from aiohttp.web import Request, Response, RouteTableDef, json_response from aiohttp_jwt import login_required -from music_assistant.helpers.util import json_serializer from music_assistant.helpers.web import async_media_items_from_body, async_stream_json routes = RouteTableDef() @@ -17,7 +17,7 @@ async def async_playlist(request: Request): if item_id is None or provider is None: return Response(text="invalid item or provider", status=501) result = await request.app["mass"].music.async_get_playlist(item_id, provider) - return Response(body=json_serializer(result), content_type="application/json") + return json_response(result) @routes.get("/api/playlists/{item_id}/tracks") @@ -37,10 +37,10 @@ async def async_playlist_tracks(request: Request): async def async_add_playlist_tracks(request: Request): """Add tracks to (editable) playlist.""" item_id = request.match_info.get("item_id") - body = await request.json() + body = await request.json(loads=ujson.loads) tracks = await async_media_items_from_body(request.app["mass"], body) result = await request.app["mass"].music.async_add_playlist_tracks(item_id, tracks) - return Response(body=json_serializer(result), content_type="application/json") + return json_response(result) @routes.delete("/api/playlists/{item_id}/tracks") @@ -48,9 +48,9 @@ async def async_add_playlist_tracks(request: Request): async def async_remove_playlist_tracks(request: Request): """Remove tracks from (editable) playlist.""" item_id = request.match_info.get("item_id") - body = await request.json() + body = await request.json(loads=ujson.loads) tracks = await async_media_items_from_body(request.app["mass"], body) result = await request.app["mass"].music.async_remove_playlist_tracks( item_id, tracks ) - return Response(body=json_serializer(result), content_type="application/json") + return json_response(result) diff --git a/music_assistant/web/endpoints/websocket.py b/music_assistant/web/endpoints/websocket.py index 13cbf486..ca6894ab 100644 --- a/music_assistant/web/endpoints/websocket.py +++ b/music_assistant/web/endpoints/websocket.py @@ -3,7 +3,7 @@ import logging import jwt -import orjson +import ujson from aiohttp import WSMsgType from aiohttp.web import Request, RouteTableDef, WebSocketResponse from music_assistant.helpers.util import json_serializer @@ -29,7 +29,7 @@ async def async_websocket_handler(request: Request): msg_details = msg_details.to_dict() ws_msg = {"message": msg, "message_details": msg_details} try: - await ws_response.send_str(json_serializer(ws_msg).decode()) + await ws_response.send_str(json_serializer(ws_msg)) # pylint: disable=broad-except except Exception as exc: LOGGER.debug( @@ -44,8 +44,8 @@ async def async_websocket_handler(request: Request): LOGGER.warning(msg.data) continue try: - data = msg.json(loads=orjson.loads) - except orjson.JSONDecodeError: + data = msg.json(loads=ujson.loads) + except ValueError: await async_send_message( "error", 'commands must be issued in json format \ diff --git a/requirements.txt b/requirements.txt index f1ee2722..78cbfcb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,5 @@ aiohttp_jwt==0.6.1 zeroconf==0.28.6 passlib==1.7.4 cryptography==3.2 -orjson==3.4.3 +ujson==4.0.1 +mashumaro==1.13 -- 2.34.1