optimize docker files, update spotty bin
authorMarcel van der Veldt <m.vanderveldt@outlook.com>
Tue, 3 Nov 2020 22:17:54 +0000 (23:17 +0100)
committerMarcel van der Veldt <m.vanderveldt@outlook.com>
Tue, 3 Nov 2020 22:17:54 +0000 (23:17 +0100)
16 files changed:
.github/workflows/docker-build-multiarch.yml
Dockerfile
Dockerfile.debian
music_assistant/providers/spotify/__init__.py
music_assistant/providers/spotify/spotty/arm-linux/spotty-hf [deleted file]
music_assistant/providers/spotify/spotty/darwin/spotty
music_assistant/providers/spotify/spotty/linux/spotty-aarch64 [new file with mode: 0755]
music_assistant/providers/spotify/spotty/linux/spotty-armhf [new file with mode: 0755]
music_assistant/providers/spotify/spotty/linux/spotty-armv6 [new file with mode: 0755]
music_assistant/providers/spotify/spotty/linux/spotty-i386 [new file with mode: 0755]
music_assistant/providers/spotify/spotty/linux/spotty-muslhf [new file with mode: 0755]
music_assistant/providers/spotify/spotty/linux/spotty-x86_64 [new file with mode: 0755]
music_assistant/providers/spotify/spotty/windows/spotty.exe
music_assistant/providers/spotify/spotty/x86-linux/spotty [deleted file]
music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64 [deleted file]
requirements.txt

index 7ce53f50b768872ac48fcad4030b0404aa10f543..8684f275d784992f4a4771ac5862f633a86a3779 100644 (file)
@@ -65,6 +65,14 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       - name: Build
         uses: docker/build-push-action@v2
         with:
@@ -73,4 +81,6 @@ jobs:
           file: ./Dockerfile
           platforms: linux/amd64,linux/arm/v7,linux/arm/v6,linux/arm64
           push: true
-          tags: ${{ steps.prep.outputs.tags }}
\ No newline at end of file
+          tags: ${{ steps.prep.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
\ No newline at end of file
index 09de752a8eba25055046179987d72f2d0274a7a9..1f05678ebf2403dae1a1f74c9e46a81ab5d32229 100755 (executable)
@@ -1,43 +1,18 @@
-FROM alpine:3.12
+FROM python:3.8-alpine3.12 AS builder
 
-ARG JEMALLOC_VERSION=5.2.1
-WORKDIR /usr/src/
-COPY . .
-
-# Install packages
-RUN set -x \
+#### BUILD DEPENDENCIES AND PYTHON WHEELS
+RUN echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
     && apk update \
-    && echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-    && echo "http://dl-8.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
-    # install default packages
-    && apk add --no-cache \
-        tzdata \
-        ca-certificates \
+    && apk add  \
         curl \
-        flac \
-        sox \
-        libuv \
-        ffmpeg \
-        python3 \
-        py3-pillow \
-        py3-numpy \
-        py3-scipy \
-        py3-aiohttp \
-        py3-jwt \
-        py3-passlib \
-        py3-cryptography \
-        py3-zeroconf \
-        py3-pytaglib \
-        py3-pip \
-    # install (temp) build packages
-    && apk add --no-cache --virtual .build-deps \
+        bind-tools \
+        ca-certificates \
+        alpine-sdk \
         build-base \
-        python3-dev \
-        libsndfile-dev \
-        taglib-dev \
         openblas-dev \
         lapack-dev \
         libffi-dev \
+        python3-dev \
         gcc \
         gfortran \
         freetype-dev \
@@ -54,21 +29,71 @@ RUN set -x \
         zlib-dev \
         libuv-dev \
         libffi-dev \
-    # setup jmalloc
-    && curl -L -f -s "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" \
-            | tar -xjf - -C /usr/src \
-        && cd /usr/src/jemalloc-${JEMALLOC_VERSION} \
-        && ./configure \
-        && make \
-        && make install \
-        && cd /usr/src \
+        # pillow deps ?
+        freetype \
+        lcms2 \
+        libimagequant \
+        libjpeg-turbo \
+        libwebp \
+        libxcb \
+        openjpeg \
+        tiff \
+        zlib \
+        taglib-dev \
+        libsndfile-dev
+
+# build jemalloc
+ARG JEMALLOC_VERSION=5.2.1
+RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 \
+        | tar -xjf - -C /tmp \
+    && cd /tmp/jemalloc-${JEMALLOC_VERSION} \
+    && ./configure \
+    && make \
+    && make install
+
+# build python wheels
+WORKDIR /wheels
+COPY ./requirements.txt /wheels/requirements.txt
+RUN pip install -U pip \
+    && pip wheel -r ./requirements.txt \
+    && pip wheel uvloop
+
+#### FINAL IMAGE
+FROM python:3.8-alpine3.12
+
+WORKDIR /usr/src/
+COPY . .
+COPY --from=builder /wheels /wheels
+COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so
+RUN set -x \
+    # Install runtime dependency packages
+    && apk add --no-cache --upgrade --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \
+        libgcc \
+        tzdata \
+        ca-certificates \
+        bind-tools \
+        curl \
+        flac \
+    && apk add --no-cache --upgrade --repository http://dl-cdn.alpinelinux.org/alpine/edge/community \
+        sox \
+        ffmpeg \
+        taglib \
+        libsndfile \
+    # Make sure pip is updated
+    pip install -U pip \
     # make sure uvloop is installed
-    &&  pip install uvloop \
+    && pip install --no-cache-dir uvloop -f /wheels \
+    # pre-install all requirements (needed for numpy/scipy)
+    && pip install --no-cache-dir -r /wheels/requirements.txt -f /wheels \
+    # Include frontend-app in the source files
+    && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \
+    && mv app-master/docs /usr/src/music_assistant/web/static \
     # install music assistant
     && python3 setup.py install \
-    # cleanup build files
-    && apk del .build-deps \
-        && rm -rf /usr/src/*
+    # cleanup
+    && rm -rf /usr/src/* \
+    && rm -rf /tmp/* \
+    && rm -rf /wheels
 
 ENV DEBUG=false
 EXPOSE 8095/tcp
index 337a0aaa9280c8556c5d1166abadacd649b81192..8089ba26b1e2cfe42e7b8a4b3ade58d9d7a5f258 100644 (file)
@@ -1,36 +1,58 @@
-FROM python:3.8-slim
+FROM python:3.8-slim as builder
+
+RUN set -x \
+    # Install packages
+    && apt-get update && apt-get install -y --no-install-recommends \
+        curl ca-certificates build-essential gcc libtag1-dev libffi-dev
 
+# build jemalloc
 ARG JEMALLOC_VERSION=5.2.1
-WORKDIR /usr/src/
-COPY . .
+RUN curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 \
+        | tar -xjf - -C /tmp \
+    && cd /tmp/jemalloc-${JEMALLOC_VERSION} \
+    && ./configure \
+    && make \
+    && make install
 
-ENV RUNTIME_DEPS="tzdata ca-certificates flac sox libsox-fmt-all ffmpeg libsndfile1 libtag1v5"
-ENV BUILD_DEPS="git curl build-essential libtag1-dev libffi-dev"
+# build python wheels
+WORKDIR /wheels
+COPY ./requirements.txt /wheels/requirements.txt
+RUN pip install -U pip \
+    && pip wheel -r ./requirements.txt \
+    && pip wheel uvloop
 
+#### FINAL IMAGE
+FROM python:3.8-slim
+
+WORKDIR /usr/src/
+COPY . .
+COPY --from=builder /wheels /wheels
+COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so
 RUN set -x \
-    # Install packages
-    && apt-get update && apt-get install -y --no-install-recommends \
-        # required packages
-        $RUNTIME_DEPS \
-        # (temp) build packages
-        $BUILD_DEPS \
-    # setup jmalloc
-    && curl -L -s https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2 | tar -xjf - -C /usr/src \
-        && cd /usr/src/jemalloc-${JEMALLOC_VERSION} \
-        && ./configure \
-        && make \
-        && make install \
-        && cd /usr/src \
+    # Install runtime dependency packages
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl tzdata ca-certificates flac sox libsox-fmt-all ffmpeg libsndfile1 libtag1v5 \
+    # Make sure pip is updated
+    && pip install -U pip \
     # make sure uvloop is installed
-    &&  pip install uvloop \
+    && pip install --no-cache-dir uvloop -f /wheels \
+    # pre-install all requirements (needed for numpy/scipy)
+    && pip install --no-cache-dir -r /wheels/requirements.txt -f /wheels \
+    # Include frontend-app in the source files
+    && curl -L https://github.com/music-assistant/app/archive/master.tar.gz | tar xz \
+    && mv app-master/docs /usr/src/music_assistant/web/static \
     # install music assistant
     && python3 setup.py install \
-    # cleanup build files
-    && apt-get remove --purge -y --allow-remove-essential $BUILD_DEPS \
+    # cleanup
+    && rm -rf /usr/src/* \
+    && rm -rf /tmp/* \
+    && rm -rf /wheels \
     && rm -rf /var/lib/apt/lists/*
 
-
 ENV DEBUG=false
+EXPOSE 8095/tcp
+
 VOLUME [ "/data" ]
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so
index daff2a82b2a71085a47721b8fca0058a27508946..c71f9bad6defecaad9e46f0ecb9ae167c098d8e3 100644 (file)
@@ -615,26 +615,32 @@ class SpotifyProvider(MusicProvider):
     @staticmethod
     def get_spotty_binary():
         """Find the correct spotty binary belonging to the platform."""
-        sp_binary = None
         if platform.system() == "Windows":
-            sp_binary = os.path.join(
+            return os.path.join(
                 os.path.dirname(__file__), "spotty", "windows", "spotty.exe"
             )
-        elif platform.system() == "Darwin":
+        if platform.system() == "Darwin":
             # macos binary is x86_64 intel
-            sp_binary = os.path.join(
-                os.path.dirname(__file__), "spotty", "darwin", "spotty"
-            )
-        elif platform.system() == "Linux":
-            # try to find out the correct architecture by trial and error
+            return os.path.join(os.path.dirname(__file__), "spotty", "darwin", "spotty")
+        if platform.system() == "Linux":
             architecture = platform.machine()
-            if architecture.startswith("AMD64") or architecture.startswith("x86_64"):
+            if architecture in ["AMD64", "x86_64"]:
                 # generic linux x86_64 binary
-                sp_binary = os.path.join(
-                    os.path.dirname(__file__), "spotty", "x86-linux", "spotty-x86_64"
+                return os.path.join(
+                    os.path.dirname(__file__), "spotty", "linux", "spotty-x86_64"
                 )
-            else:
-                sp_binary = os.path.join(
-                    os.path.dirname(__file__), "spotty", "arm-linux", "spotty-muslhf"
+            if "i386" in architecture:
+                # i386 linux binary
+                return os.path.join(
+                    os.path.dirname(__file__), "spotty", "linux", "spotty-i386"
+                )
+            if "aarch64" in architecture or "armv8" in architecture:
+                # arm64 linux binary
+                return os.path.join(
+                    os.path.dirname(__file__), "spotty", "x86-linux", "spotty-aarch64"
                 )
-        return sp_binary
+            # assume armv7
+            return os.path.join(
+                os.path.dirname(__file__), "spotty", "arm-linux", "spotty-armhf"
+            )
+        return None
diff --git a/music_assistant/providers/spotify/spotty/arm-linux/spotty-hf b/music_assistant/providers/spotify/spotty/arm-linux/spotty-hf
deleted file mode 100755 (executable)
index c928d8a..0000000
Binary files a/music_assistant/providers/spotify/spotty/arm-linux/spotty-hf and /dev/null differ
index 44c6b6046c6d396fd594be02f9d442d56c72256a..e20a0d910175605e2a2f36a15483ed1e6517b808 100755 (executable)
Binary files a/music_assistant/providers/spotify/spotty/darwin/spotty and b/music_assistant/providers/spotify/spotty/darwin/spotty differ
diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-aarch64 b/music_assistant/providers/spotify/spotty/linux/spotty-aarch64
new file mode 100755 (executable)
index 0000000..e48b8dc
Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-aarch64 differ
diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-armhf b/music_assistant/providers/spotify/spotty/linux/spotty-armhf
new file mode 100755 (executable)
index 0000000..bee2f66
Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-armhf differ
diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-armv6 b/music_assistant/providers/spotify/spotty/linux/spotty-armv6
new file mode 100755 (executable)
index 0000000..3b474e5
Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-armv6 differ
diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-i386 b/music_assistant/providers/spotify/spotty/linux/spotty-i386
new file mode 100755 (executable)
index 0000000..68c7dd0
Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-i386 differ
diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-muslhf b/music_assistant/providers/spotify/spotty/linux/spotty-muslhf
new file mode 100755 (executable)
index 0000000..c172724
Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-muslhf differ
diff --git a/music_assistant/providers/spotify/spotty/linux/spotty-x86_64 b/music_assistant/providers/spotify/spotty/linux/spotty-x86_64
new file mode 100755 (executable)
index 0000000..5aff94b
Binary files /dev/null and b/music_assistant/providers/spotify/spotty/linux/spotty-x86_64 differ
index 6ce9b19e3cdb7a154554f24942d60fb50d5ebc02..0ccbddf3cced11e0483b4820d3a871070da93ca0 100755 (executable)
Binary files a/music_assistant/providers/spotify/spotty/windows/spotty.exe and b/music_assistant/providers/spotify/spotty/windows/spotty.exe differ
diff --git a/music_assistant/providers/spotify/spotty/x86-linux/spotty b/music_assistant/providers/spotify/spotty/x86-linux/spotty
deleted file mode 100755 (executable)
index b2c3f34..0000000
Binary files a/music_assistant/providers/spotify/spotty/x86-linux/spotty and /dev/null differ
diff --git a/music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64 b/music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64
deleted file mode 100755 (executable)
index 58911cf..0000000
Binary files a/music_assistant/providers/spotify/spotty/x86-linux/spotty-x86_64 and /dev/null differ
index 8b60d598b7573ad4c7758d90f50678d397b5d963..a3e4e74c304a1e848e76b147d3d12528c042fb88 100644 (file)
@@ -1,23 +1,23 @@
 argparse==1.4.0
-aiohttp[speedups]~=3.7.2
+aiohttp[speedups]==3.7.2
 pychromecast==7.5.1
 asyncio-throttle==1.0.1
 aiofile==3.1.0
 aiosqlite==0.16.0
-pytaglib~=1.4.6
+pytaglib==1.4.6
 python-slugify==4.0.1
 memory-tempfile==2.2.3
 pyloudnorm==0.1.0
 SoundFile==0.10.3.post1
 aiorun==2020.11.1
 soco==0.20
-pillow<=8.0.1
+pillow==8.0.1
 aiohttp_cors==0.7.0
 unidecode==1.1.1
-PyJWT~=1.7.1
+PyJWT==1.7.1
 aiohttp_jwt==0.6.1
-zeroconf>=0.28.5
-passlib~=1.7.4
-cryptography>=2.9.2
+zeroconf==0.28.6
+passlib==1.7.4
+cryptography==3.2
 ujson==4.0.1
 mashumaro==1.13