From: Marcel van der Veldt Date: Tue, 3 Nov 2020 22:17:54 +0000 (+0100) Subject: optimize docker files, update spotty bin X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=0db940b05a6cc3a9078663d6410fdceec971eae2;p=music-assistant-server.git optimize docker files, update spotty bin --- diff --git a/.github/workflows/docker-build-multiarch.yml b/.github/workflows/docker-build-multiarch.yml index 7ce53f50..8684f275 100644 --- a/.github/workflows/docker-build-multiarch.yml +++ b/.github/workflows/docker-build-multiarch.yml @@ -65,6 +65,14 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Build uses: docker/build-push-action@v2 with: @@ -73,4 +81,6 @@ jobs: file: ./Dockerfile platforms: linux/amd64,linux/arm/v7,linux/arm/v6,linux/arm64 push: true - tags: ${{ steps.prep.outputs.tags }} \ No newline at end of file + tags: ${{ steps.prep.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 09de752a..1f05678e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,18 @@ -FROM alpine:3.12 +FROM python:3.8-alpine3.12 AS builder -ARG JEMALLOC_VERSION=5.2.1 -WORKDIR /usr/src/ -COPY . . - -# Install packages -RUN set -x \ +#### BUILD DEPENDENCIES AND PYTHON WHEELS +RUN echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ && apk update \ - && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ - && echo "http://dl-8.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ - # install default packages - && apk add --no-cache \ - tzdata \ - ca-certificates \ + && apk add \ curl \ - flac \ - sox \ - libuv \ - ffmpeg \ - python3 \ - py3-pillow \ - py3-numpy \ - py3-scipy \ - py3-aiohttp \ - py3-jwt \ - py3-passlib \ - py3-cryptography \ - py3-zeroconf \ - py3-pytaglib \ - py3-pip \ - # install (temp) build packages - && apk add --no-cache --virtual .build-deps \ + bind-tools \ + ca-certificates \ + alpine-sdk \ build-base \ - python3-dev \ - libsndfile-dev \ - taglib-dev \ openblas-dev \ lapack-dev \ libffi-dev \ + python3-dev \ gcc \ gfortran \ freetype-dev \ @@ -54,21 +29,71 @@ RUN set -x \ zlib-dev \ libuv-dev \ libffi-dev \ - # setup jmalloc - && curl -L -f -s "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" \ - | tar -xjf - -C /usr/src \ - && cd /usr/src/jemalloc-${JEMALLOC_VERSION} \ - && ./configure \ - && make \ - && make install \ - && cd /usr/src \ + # pillow deps ? + freetype \ + lcms2 \ + libimagequant \ + libjpeg-turbo \ + libwebp \ + libxcb \ + openjpeg \ + tiff \ + zlib \ + taglib-dev \ + libsndfile-dev + +# build jemalloc +ARG JEMALLOC_VERSION=5.2.1 +RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 \ + | tar -xjf - -C /tmp \ + && cd /tmp/jemalloc-${JEMALLOC_VERSION} \ + && ./configure \ + && make \ + && make install + +# build python wheels +WORKDIR /wheels +COPY ./requirements.txt /wheels/requirements.txt +RUN pip install -U pip \ + && pip wheel -r ./requirements.txt \ + && pip wheel uvloop + +#### FINAL IMAGE +FROM python:3.8-alpine3.12 + +WORKDIR /usr/src/ +COPY . . +COPY --from=builder /wheels /wheels +COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so +RUN set -x \ + # Install runtime dependency packages + && apk add --no-cache --upgrade --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ + libgcc \ + tzdata \ + ca-certificates \ + bind-tools \ + curl \ + flac \ + && apk add --no-cache --upgrade --repository http://dl-cdn.alpinelinux.org/alpine/edge/community \ + sox \ + ffmpeg \ + taglib \ + libsndfile \ + # Make sure pip is updated + pip install -U pip \ # make sure uvloop is installed - && pip install uvloop \ + && pip install --no-cache-dir uvloop -f /wheels \ + # pre-install all requirements (needed for numpy/scipy) + && pip install --no-cache-dir -r /wheels/requirements.txt -f /wheels \ + # Include frontend-app in the source files + && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \ + && mv app-master/docs /usr/src/music_assistant/web/static \ # install music assistant && python3 setup.py install \ - # cleanup build files - && apk del .build-deps \ - && rm -rf /usr/src/* + # cleanup + && rm -rf /usr/src/* \ + && rm -rf /tmp/* \ + && rm -rf /wheels ENV DEBUG=false EXPOSE 8095/tcp diff --git a/Dockerfile.debian b/Dockerfile.debian index 337a0aaa..8089ba26 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,36 +1,58 @@ -FROM python:3.8-slim +FROM python:3.8-slim as builder + +RUN set -x \ + # Install packages + && apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates build-essential gcc libtag1-dev libffi-dev +# build jemalloc ARG JEMALLOC_VERSION=5.2.1 -WORKDIR /usr/src/ -COPY . . +RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 \ + | tar -xjf - -C /tmp \ + && cd /tmp/jemalloc-${JEMALLOC_VERSION} \ + && ./configure \ + && make \ + && make install -ENV RUNTIME_DEPS="tzdata ca-certificates flac sox libsox-fmt-all ffmpeg libsndfile1 libtag1v5" -ENV BUILD_DEPS="git curl build-essential libtag1-dev libffi-dev" +# build python wheels +WORKDIR /wheels +COPY ./requirements.txt /wheels/requirements.txt +RUN pip install -U pip \ + && pip wheel -r ./requirements.txt \ + && pip wheel uvloop +#### FINAL IMAGE +FROM python:3.8-slim + +WORKDIR /usr/src/ +COPY . . +COPY --from=builder /wheels /wheels +COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so RUN set -x \ - # Install packages - && apt-get update && apt-get install -y --no-install-recommends \ - # required packages - $RUNTIME_DEPS \ - # (temp) build packages - $BUILD_DEPS \ - # setup jmalloc - && curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 | tar -xjf - -C /usr/src \ - && cd /usr/src/jemalloc-${JEMALLOC_VERSION} \ - && ./configure \ - && make \ - && make install \ - && cd /usr/src \ + # Install runtime dependency packages + && apt-get update \ + && apt-get install -y --no-install-recommends \ + curl tzdata ca-certificates flac sox libsox-fmt-all ffmpeg libsndfile1 libtag1v5 \ + # Make sure pip is updated + && pip install -U pip \ # make sure uvloop is installed - && pip install uvloop \ + && pip install --no-cache-dir uvloop -f /wheels \ + # pre-install all requirements (needed for numpy/scipy) + && pip install --no-cache-dir -r /wheels/requirements.txt -f /wheels \ + # Include frontend-app in the source files + && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \ + && mv app-master/docs /usr/src/music_assistant/web/static \ # install music assistant && python3 setup.py install \ - # cleanup build files - && apt-get remove --purge -y --allow-remove-essential $BUILD_DEPS \ + # cleanup + && rm -rf /usr/src/* \ + && rm -rf /tmp/* \ + && rm -rf /wheels \ && rm -rf /var/lib/apt/lists/* - ENV DEBUG=false +EXPOSE 8095/tcp + VOLUME [ "/data" ] ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so diff --git a/music_assistant/providers/spotify/__init__.py b/music_assistant/providers/spotify/__init__.py index daff2a82..c71f9bad 100644 --- a/music_assistant/providers/spotify/__init__.py +++ b/music_assistant/providers/spotify/__init__.py @@ -615,26 +615,32 @@ class SpotifyProvider(MusicProvider): @staticmethod def get_spotty_binary(): """Find the correct spotty binary belonging to the platform.""" - sp_binary = None if platform.system() == "Windows": - sp_binary = os.path.join( + return os.path.join( os.path.dirname(__file__), "spotty", "windows", "spotty.exe" ) - elif platform.system() == "Darwin": + if platform.system() == "Darwin": # macos binary is x86_64 intel - sp_binary = os.path.join( - os.path.dirname(__file__), "spotty", "darwin", "spotty" - ) - elif platform.system() == "Linux": - # try to find out the correct architecture by trial and error + return os.path.join(os.path.dirname(__file__), "spotty", "darwin", "spotty") + if platform.system() == "Linux": architecture = platform.machine() - if architecture.startswith("AMD64") or architecture.startswith("x86_64"): + if architecture in ["AMD64", "x86_64"]: # generic linux x86_64 binary - sp_binary = os.path.join( - os.path.dirname(__file__), "spotty", "x86-linux", "spotty-x86_64" + return os.path.join( + os.path.dirname(__file__), "spotty", "linux", "spotty-x86_64" ) - else: - sp_binary = os.path.join( - os.path.dirname(__file__), "spotty", "arm-linux", "spotty-muslhf" + if "i386" in architecture: + # i386 linux binary + return os.path.join( + os.path.dirname(__file__), "spotty", "linux", "spotty-i386" + ) + if "aarch64" in architecture or "armv8" in architecture: + # arm64 linux binary + return os.path.join( + os.path.dirname(__file__), "spotty", "x86-linux", "spotty-aarch64" ) - return sp_binary + # assume armv7 + return os.path.join( + os.path.dirname(__file__), "spotty", "arm-linux", "spotty-armhf" + ) + return None diff --git a/music_assistant/providers/spotify/spotty/arm-linux/spotty-hf b/music_assistant/providers/spotify/spotty/arm-linux/spotty-hf deleted file mode 100755 index c928d8a8..00000000 Binary files a/music_assistant/providers/spotify/spotty/arm-linux/spotty-hf and /dev/null differ diff --git a/music_assistant/providers/spotify/spotty/darwin/spotty b/music_assistant/providers/spotify/spotty/darwin/spotty index 44c6b604..e20a0d91 100755 Binary files a/music_assistant/providers/spotify/spotty/darwin/spotty and b/music_assistant/providers/spotify/spotty/darwin/spotty differ diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-aarch64 b/music_assistant/providers/spotify/spotty/linux/spotty-aarch64 new file mode 100755 index 00000000..e48b8dc8 Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-aarch64 differ diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-armhf b/music_assistant/providers/spotify/spotty/linux/spotty-armhf new file mode 100755 index 00000000..bee2f665 Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-armhf differ diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-armv6 b/music_assistant/providers/spotify/spotty/linux/spotty-armv6 new file mode 100755 index 00000000..3b474e5d Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-armv6 differ diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-i386 b/music_assistant/providers/spotify/spotty/linux/spotty-i386 new file mode 100755 index 00000000..68c7dd06 Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-i386 differ diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-muslhf b/music_assistant/providers/spotify/spotty/linux/spotty-muslhf new file mode 100755 index 00000000..c172724f Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-muslhf differ diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-x86_64 b/music_assistant/providers/spotify/spotty/linux/spotty-x86_64 new file mode 100755 index 00000000..5aff94b6 Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-x86_64 differ diff --git a/music_assistant/providers/spotify/spotty/windows/spotty.exe b/music_assistant/providers/spotify/spotty/windows/spotty.exe index 6ce9b19e..0ccbddf3 100755 Binary files a/music_assistant/providers/spotify/spotty/windows/spotty.exe and b/music_assistant/providers/spotify/spotty/windows/spotty.exe differ diff --git a/music_assistant/providers/spotify/spotty/x86-linux/spotty b/music_assistant/providers/spotify/spotty/x86-linux/spotty deleted file mode 100755 index b2c3f349..00000000 Binary files a/music_assistant/providers/spotify/spotty/x86-linux/spotty and /dev/null differ diff --git a/music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64 b/music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64 deleted file mode 100755 index 58911cf5..00000000 Binary files a/music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64 and /dev/null differ diff --git a/requirements.txt b/requirements.txt index 8b60d598..a3e4e74c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,23 @@ argparse==1.4.0 -aiohttp[speedups]~=3.7.2 +aiohttp[speedups]==3.7.2 pychromecast==7.5.1 asyncio-throttle==1.0.1 aiofile==3.1.0 aiosqlite==0.16.0 -pytaglib~=1.4.6 +pytaglib==1.4.6 python-slugify==4.0.1 memory-tempfile==2.2.3 pyloudnorm==0.1.0 SoundFile==0.10.3.post1 aiorun==2020.11.1 soco==0.20 -pillow<=8.0.1 +pillow==8.0.1 aiohttp_cors==0.7.0 unidecode==1.1.1 -PyJWT~=1.7.1 +PyJWT==1.7.1 aiohttp_jwt==0.6.1 -zeroconf>=0.28.5 -passlib~=1.7.4 -cryptography>=2.9.2 +zeroconf==0.28.6 +passlib==1.7.4 +cryptography==3.2 ujson==4.0.1 mashumaro==1.13