From: Marcel van der Veldt Date: Wed, 4 Nov 2020 17:02:29 +0000 (+0100) Subject: drop dumpy and scipy deps X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=01ca739ce106043330e6a225021c74e13c9f86da;p=music-assistant-server.git drop dumpy and scipy deps --- diff --git a/Dockerfile b/Dockerfile index 1f05678e..eaa90307 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,41 @@ -FROM python:3.8-alpine3.12 AS builder +FROM python:3.8-alpine3.12 + +ARG JEMALLOC_VERSION=5.2.1 +WORKDIR /tmp +COPY . . -#### BUILD DEPENDENCIES AND PYTHON WHEELS -RUN echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ +# Install packages +RUN set -x \ && apk update \ - && apk add \ - curl \ - bind-tools \ + && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ + && echo "http://dl-8.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ + # install default packages + && apk add --no-cache \ + tzdata \ ca-certificates \ - alpine-sdk \ + curl \ + flac \ + sox \ + libuv \ + ffmpeg \ + uchardet \ + # dependencies for pillow + freetype \ + lcms2 \ + libimagequant \ + libjpeg-turbo \ + libwebp \ + libxcb \ + openjpeg \ + tiff \ + zlib \ + # install (temp) build packages + && apk add --no-cache --virtual .build-deps \ build-base \ - openblas-dev \ - lapack-dev \ - libffi-dev \ - python3-dev \ + libsndfile-dev \ + taglib-dev \ gcc \ - gfortran \ + musl-dev \ freetype-dev \ libpng-dev \ libressl-dev \ @@ -29,71 +50,22 @@ RUN echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/reposit zlib-dev \ libuv-dev \ libffi-dev \ - # pillow deps ? - freetype \ - lcms2 \ - libimagequant \ - libjpeg-turbo \ - libwebp \ - libxcb \ - openjpeg \ - tiff \ - zlib \ - taglib-dev \ - libsndfile-dev - -# build jemalloc -ARG JEMALLOC_VERSION=5.2.1 -RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 \ - | tar -xjf - -C /tmp \ - && cd /tmp/jemalloc-${JEMALLOC_VERSION} \ - && ./configure \ - && make \ - && make install - -# build python wheels -WORKDIR /wheels -COPY ./requirements.txt /wheels/requirements.txt -RUN pip install -U pip \ - && pip wheel -r ./requirements.txt \ - && pip wheel uvloop - -#### FINAL IMAGE -FROM python:3.8-alpine3.12 - -WORKDIR /usr/src/ -COPY . . -COPY --from=builder /wheels /wheels -COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so -RUN set -x \ - # Install runtime dependency packages - && apk add --no-cache --upgrade --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ - libgcc \ - tzdata \ - ca-certificates \ - bind-tools \ - curl \ - flac \ - && apk add --no-cache --upgrade --repository http://dl-cdn.alpinelinux.org/alpine/edge/community \ - sox \ - ffmpeg \ - taglib \ - libsndfile \ - # Make sure pip is updated - pip install -U pip \ - # make sure uvloop is installed - && pip install --no-cache-dir uvloop -f /wheels \ - # pre-install all requirements (needed for numpy/scipy) - && pip install --no-cache-dir -r /wheels/requirements.txt -f /wheels \ - # Include frontend-app in the source files - && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \ - && mv app-master/docs /usr/src/music_assistant/web/static \ + uchardet-dev \ + # setup jemalloc + && curl -L -f -s "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" \ + | tar -xjf - -C /tmp \ + && cd /tmp/jemalloc-${JEMALLOC_VERSION} \ + && ./configure \ + && make \ + && make install \ + && cd /tmp \ + # make sure optional packages are installed + && pip install uvloop cchardet aiodns brotlipy \ # install music assistant - && python3 setup.py install \ - # cleanup - && rm -rf /usr/src/* \ - && rm -rf /tmp/* \ - && rm -rf /wheels + && pip install . \ + # cleanup build files + && apk del .build-deps \ + && rm -rf /tmp/* ENV DEBUG=false EXPOSE 8095/tcp diff --git a/Dockerfile.debian b/Dockerfile.debian index 8089ba26..d74db0a1 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,9 +1,14 @@ FROM python:3.8-slim as builder RUN set -x \ - # Install packages + # Install buildtime packages && apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates build-essential gcc libtag1-dev libffi-dev + curl \ + ca-certificates \ + build-essential \ + gcc \ + libtag1-dev \ + libffi-dev # build jemalloc ARG JEMALLOC_VERSION=5.2.1 @@ -16,36 +21,35 @@ RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC # build python wheels WORKDIR /wheels -COPY ./requirements.txt /wheels/requirements.txt -RUN pip install -U pip \ - && pip wheel -r ./requirements.txt \ - && pip wheel uvloop - +COPY . /tmp +RUN pip wheel uvloop cchardet aiodns brotlipy \ + && pip wheel --extra-index-url=https://www.piwheels.org/simple -r /tmp/requirements.txt \ + # Include frontend-app in the source files + && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \ + && mv app-master/docs /tmp/music_assistant/web/static \ + && pip wheel --extra-index-url=https://www.piwheels.org/simple /tmp + #### FINAL IMAGE FROM python:3.8-slim -WORKDIR /usr/src/ -COPY . . +WORKDIR /wheels COPY --from=builder /wheels /wheels COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so RUN set -x \ # Install runtime dependency packages && apt-get update \ && apt-get install -y --no-install-recommends \ - curl tzdata ca-certificates flac sox libsox-fmt-all ffmpeg libsndfile1 libtag1v5 \ - # Make sure pip is updated - && pip install -U pip \ - # make sure uvloop is installed - && pip install --no-cache-dir uvloop -f /wheels \ - # pre-install all requirements (needed for numpy/scipy) - && pip install --no-cache-dir -r /wheels/requirements.txt -f /wheels \ - # Include frontend-app in the source files - && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \ - && mv app-master/docs /usr/src/music_assistant/web/static \ - # install music assistant - && python3 setup.py install \ + curl \ + tzdata \ + ca-certificates \ + flac \ + sox \ + libsox-fmt-all \ + ffmpeg \ + libtag1v5 \ + # install music assistant (and all it's dependencies) using the prebuilt wheels + && pip install --no-cache-dir -f /wheels --extra-index-url=https://www.piwheels.org/simple music_assistant \ # cleanup - && rm -rf /usr/src/* \ && rm -rf /tmp/* \ && rm -rf /wheels \ && rm -rf /var/lib/apt/lists/* diff --git a/music_assistant/managers/streams.py b/music_assistant/managers/streams.py index 39f6d813..3b300d8d 100755 --- a/music_assistant/managers/streams.py +++ b/music_assistant/managers/streams.py @@ -8,14 +8,12 @@ All audio is processed by the SoX executable, using various subprocess streams. """ import asyncio import gc -import io import logging import shlex +import subprocess from enum import Enum from typing import AsyncGenerator, List, Optional, Tuple -import pyloudnorm -import soundfile from aiofile import AIOFile, Reader from music_assistant.constants import ( CONF_MAX_SAMPLE_RATE, @@ -451,12 +449,15 @@ class StreamManager: if track_loudness is None: # only when needed we do the analyze stuff LOGGER.debug("Start analyzing track %s", item_key) - # calculate BS.1770 R128 integrated loudness - with io.BytesIO(audio_data) as tmpfile: - data, rate = soundfile.read(tmpfile) - meter = pyloudnorm.Meter(rate) # create BS.1770 meter - loudness = meter.integrated_loudness(data) # measure loudness - del data + # calculate BS.1770 R128 integrated loudness with ffmpeg + # we used pyloudnorm here before but the numpy/scipy requirements were too heavy, + # considered the same feature is also included in ffmpeg + value = subprocess.check_output( + "ffmpeg -i pipe: -af ebur128=framelog=verbose -f null - 2>&1 | awk '/I:/{print $2}'", + shell=True, + input=audio_data, + ) + loudness = float(value.decode().strip()) self.mass.add_job( self.mass.database.async_set_track_loudness( streamdetails.item_id, streamdetails.provider, loudness diff --git a/music_assistant/web/endpoints/websocket.py b/music_assistant/web/endpoints/websocket.py index ca6894ab..b7e9aa00 100644 --- a/music_assistant/web/endpoints/websocket.py +++ b/music_assistant/web/endpoints/websocket.py @@ -1,32 +1,47 @@ """Websocket API endpoint.""" import logging +from typing import Union import jwt import ujson from aiohttp import WSMsgType from aiohttp.web import Request, RouteTableDef, WebSocketResponse +from music_assistant.helpers.typing import MusicAssistantType from music_assistant.helpers.util import json_serializer routes = RouteTableDef() +ws_commands = dict() LOGGER = logging.getLogger("web.endpoints.websocket") +def ws_command(cmd): + """Register a websocket command.""" + + def decorate(func): + ws_commands[cmd] = func + return func + + return decorate + + @routes.get("/ws") async def async_websocket_handler(request: Request): """Handle websockets connection.""" ws_response = None authenticated = False - remove_callbacks = [] + _callbacks = [] + mass = request.app["mass"] try: ws_response = WebSocketResponse() await ws_response.prepare(request) # callback for internal events - async def async_send_message(msg, msg_details=None): - if hasattr(msg_details, "to_dict"): - msg_details = msg_details.to_dict() + async def async_send_message( + msg: str, msg_details: Union[None, dict, str] = None + ): + """Send message (back) to websocket client.""" ws_msg = {"message": msg, "message_details": msg_details} try: await ws_response.send_str(json_serializer(ws_msg)) @@ -40,61 +55,62 @@ async def async_websocket_handler(request: Request): # process incoming messages async for msg in ws_response: if msg.type != WSMsgType.TEXT: - # not sure when/if this happens but log it anyway - LOGGER.warning(msg.data) continue try: data = msg.json(loads=ujson.loads) - except ValueError: + msg = data["message"] + msg_details = data["message_details"] + except (KeyError, ValueError): await async_send_message( "error", 'commands must be issued in json format \ {"message": "command", "message_details":" optional details"}', ) continue - msg = data.get("message") - msg_details = data.get("message_details") if not authenticated and not msg == "login": # make sure client is authenticated await async_send_message("error", "authentication required") - elif msg == "login" and msg_details: - # authenticate with token + elif msg == "login": + # handle login with token try: - token_info = jwt.decode( - msg_details, request.app["mass"].web.device_id - ) - except jwt.InvalidTokenError as exc: - LOGGER.exception(exc, exc_info=exc) - error_msg = "Invalid authorization token, " + str(exc) - await async_send_message("error", error_msg) - else: - authenticated = True + token_info = jwt.decode(msg_details, mass.web.device_id) await async_send_message("login", token_info) - elif msg == "add_event_listener": - remove_callbacks.append( - request.app["mass"].add_event_listener( - async_send_message, msg_details + authenticated = True + except jwt.InvalidTokenError as exc: + async_send_message( + "error", "Invalid authorization token, " + str(exc) ) + authenticated = False + elif msg in ws_commands: + res = await ws_commands[msg](mass, msg_details) + if res is not None: + await async_send_message(res) + elif msg == "add_event_listener": + _callbacks.append( + mass.add_event_listener(async_send_message, msg_details) ) await async_send_message("event listener subscribed", msg_details) - elif msg == "player_command": - player_id = msg_details.get("player_id") - cmd = msg_details.get("cmd") - cmd_args = msg_details.get("cmd_args") - player_cmd = getattr( - request.app["mass"].players, f"async_cmd_{cmd}", None - ) - if player_cmd and cmd_args is not None: - result = await player_cmd(player_id, cmd_args) - elif player_cmd: - result = await player_cmd(player_id) - msg_details = {"cmd": cmd, "result": result} - await async_send_message("player_command_result", msg_details) + else: # simply echo the message on the eventbus request.app["mass"].signal_event(msg, msg_details) finally: LOGGER.debug("Websocket disconnected") - for remove_callback in remove_callbacks: + for remove_callback in _callbacks: remove_callback() return ws_response + + +@ws_command("player_command") +async def async_player_command(mass: MusicAssistantType, msg_details: dict): + """Handle player command.""" + player_id = msg_details.get("player_id") + cmd = msg_details.get("cmd") + cmd_args = msg_details.get("cmd_args") + player_cmd = getattr(mass.players, f"async_cmd_{cmd}", None) + if player_cmd and cmd_args is not None: + result = await player_cmd(player_id, cmd_args) + elif player_cmd: + result = await player_cmd(player_id) + msg_details = {"cmd": cmd, "result": result} + return msg_details diff --git a/requirements.txt b/requirements.txt index a3e4e74c..b89449ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,8 +7,6 @@ aiosqlite==0.16.0 pytaglib==1.4.6 python-slugify==4.0.1 memory-tempfile==2.2.3 -pyloudnorm==0.1.0 -SoundFile==0.10.3.post1 aiorun==2020.11.1 soco==0.20 pillow==8.0.1