build wheels in docker
authorMarcel van der Veldt <m.vanderveldt@outlook.com>
Thu, 9 Mar 2023 14:00:05 +0000 (15:00 +0100)
committerMarcel van der Veldt <m.vanderveldt@outlook.com>
Thu, 9 Mar 2023 14:00:05 +0000 (15:00 +0100)
.github/workflows/docker-build.yml
.pre-commit-config.yaml
Dockerfile
README.md
requirements_all.txt [new file with mode: 0644]
script/gen_requirements_all.py [new file with mode: 0644]
script/run-in-env.sh [new file with mode: 0755]

index 75ab08e24bd2c8dcce89fd03a1d084594e0cd65e..7a7414c72c1e47f3492c823461b26bcefc4322ac 100644 (file)
@@ -59,13 +59,20 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Log in to the Github Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.API_TOKEN_GITHUB }}
+
       - name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           builder: ${{ steps.buildx.outputs.name }}
           context: .
index 9ee2805a058a76c2d142d326af9ab167d2bb05ce..5e064d2ab8db630de6bc62dba0981ef21c1cb253 100644 (file)
@@ -30,9 +30,18 @@ repos:
         additional_dependencies:
           - tomli
 
-  # - repo: https://github.com/pre-commit/mirrors-mypy
-  #   rev: v0.990
-  #   hooks:
-  #     - id: mypy
-  #       additional_dependencies: [types-all]
-  #       exclude: ^examples/
+  - repo: local
+    hooks:
+      # - id: pylint
+      #   name: pylint
+      #   entry: script/run-in-env.sh pylint -j 0 --ignore-missing-annotations=y
+      #   language: script
+      #   types: [python]
+      #   files: ^music_assistant/.+\.py$
+      - id: gen_requirements_all
+        name: gen_requirements_all
+        entry: script/run-in-env.sh python3 -m script.gen_requirements_all
+        pass_filenames: false
+        language: script
+        types: [text]
+        files: ^(music_assistant/.+/manifest\.json|pyproject\.toml|\.pre-commit-config\.yaml|script/gen_requirements_all\.py)$
index 5e6b07dc4eef78463ba240a19c5c977e39a11c26..ece48b72417624759c2a26f3aa75f1191894ae10 100644 (file)
@@ -8,55 +8,52 @@ ARG PYTHON_VERSION="3.11"
 # Build Wheels                                                      #
 #                                                                   #
 #####################################################################
-FROM python:${PYTHON_VERSION}-slim as wheels-builder
+FROM python:${PYTHON_VERSION}-alpine as wheels-builder
 ARG HASS_ARCH
 
-ENV PIP_EXTRA_INDEX_URL=https://www.piwheels.org/simple
 ENV PATH="${PATH}:/root/.cargo/bin"
 
 # Install buildtime packages
 RUN set -x \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends \
-        build-essential \
-        ca-certificates \
-        curl \
-        gcc \
+    && apk add --no-cache \
+        alpine-sdk \
+        patchelf \
+        build-base \
+        cmake \
         git \
+        linux-headers \
+        autoconf \
+        automake \
+        cargo \
+        libffi \
         libffi-dev \
-        libssl-dev
-
-RUN set -x \
-    \
-    && if [ ${HASS_ARCH} = "amd64" ]; then RUST_ARCH="x86_64-unknown-linux-gnu"; fi \
-    && if [ ${HASS_ARCH} = "armv7" ]; then RUST_ARCH="armv7-unknown-linux-gnueabihf"; fi \
-    && if [ ${HASS_ARCH} = "aarch64" ]; then RUST_ARCH="aarch64-unknown-linux-gnu"; fi \
-    \
-    && curl -o rustup-init https://static.rust-lang.org/rustup/dist/${RUST_ARCH}/rustup-init \
-    && chmod +x rustup-init \
-    && ./rustup-init -y --no-modify-path --profile minimal --default-host ${RUST_ARCH}
+        git
 
 WORKDIR /wheels
-COPY requirements.txt .
+COPY requirements_all.txt .
 
-# build python wheels
+# build python wheels for all dependencies
 RUN set -x \
-    && pip wheel -r .[server]
+    && pip install --upgrade pip \
+    && pip install build \
+    && pip wheel -r requirements_all.txt --find-links "https://wheels.home-assistant.io/musllinux/"
 
+# build music assistant wheel
+COPY music_assistant music_assistant
+COPY pyproject.toml .
+COPY MANIFEST.in .
+RUN python3 -m build --wheel --outdir /wheels --skip-dependency-check
 
 #####################################################################
 #                                                                   #
 # Final Image                                                       #
 #                                                                   #
 #####################################################################
-FROM python:${PYTHON_VERSION}-slim AS final-build
+FROM python:${PYTHON_VERSION}-alpine AS final-build
 WORKDIR /app
 
-ENV DEBIAN_FRONTEND="noninteractive"
-
 RUN set -x \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends \
+    && apk add --no-cache \
         ca-certificates \
         curl \
         git \
@@ -75,7 +72,8 @@ RUN set -x \
 # Install all built wheels
 RUN --mount=type=bind,target=/tmp/wheels,source=/wheels,from=wheels-builder,rw \
     set -x \
-    && pip install --no-cache-dir -f /tmp/wheels -r /tmp/wheels/requirements.txt
+    && pip install --upgrade pip \
+    && pip install --no-cache-dir /tmp/wheels/*.whl
 
 # Required to persist build arg
 ARG BUILD_VERSION
index 692dc720fd2e5fa9616d9ca79361297231d07885..dfd56f76cecbb82b57b2f846aa82cd8a8a967e25 100644 (file)
--- a/README.md
+++ b/README.md
@@ -3,10 +3,7 @@ Music Assistant
 
 [![pypi_badge](https://img.shields.io/pypi/v/music_assistant.svg)](https://pypi.python.org/pypi/music_assistant)
 
-**Music Assistant**
+**Music Assistant Server**
 
 
-Python library for managing online and offline music sources and orchestrate playback to devices.
-
-This library has been created mainly for the [Home Assistant Integration](https://github.com/music-assistant/hass-music-assistant) but can be used stand-alone as well in other projects.
-See the examples folder for some simple code samples or have a look at the [Home Assistant Integration](https://github.com/music-assistant/hass-music-assistant) for discovering the full potential.
+Music Assistant is a free, opensource Media library manager that connects to your streaming services and a wide range of connected speakers. The server is the beating heart, the core of Music Assistant and must run on an always-on device like a Raspberry Pi, a NAS or an Intel NUC or alike.
diff --git a/requirements_all.txt b/requirements_all.txt
new file mode 100644 (file)
index 0000000..2e35199
--- /dev/null
@@ -0,0 +1,35 @@
+# WARNING: this file is autogenerated!
+
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiorun==2022.11.1
+aioslimproto==2.2.0
+aiosqlite==0.18.0
+async-upnp-client==0.33.1
+asyncio-throttle==1.0.2
+black==23.1.0
+codespell==2.2.2
+cryptography==39.0.2
+databases==0.7.0
+getmac==0.8.2
+mashumaro==3.5
+memory-tempfile==2.2.3
+music-assistant-frontend==20230308.0
+mypy==1.0.1
+orjson==3.8.6
+pillow==9.4.0
+pre-commit==2.20.0
+PyChromecast==13.0.4
+pysmb==1.2.9.1
+pytest==7.2.1
+pytest-aiohttp==1.0.4
+pytest-cov==4.0.0
+python-slugify==7.0.0
+pytube==12.1.2
+ruff==0.0.254
+shortuuid==1.0.11
+soco==0.29.1
+unidecode==1.3.6
+xmltodict==0.13.0
+ytmusicapi==0.25.0
+zeroconf==0.47.3
diff --git a/script/gen_requirements_all.py b/script/gen_requirements_all.py
new file mode 100644 (file)
index 0000000..c696ee7
--- /dev/null
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Generate updated constraint and requirements files."""
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+PACKAGE_REGEX = re.compile(r"^(?:--.+\s)?([-_\.\w\d]+).*==.+$")
+
+
+def gather_core_requirements() -> list[str]:
+    """Gather core requirements out of pyproject.toml."""
+    with open("pyproject.toml", "rb") as fp:
+        data = tomllib.load(fp)
+    dependencies: list[str] = data["project"]["optional-dependencies"]["server"]
+    dependencies += data["project"]["dependencies"]
+    dependencies += data["project"]["optional-dependencies"]["test"]
+    return dependencies
+
+
+def gather_requirements_from_manifests() -> list[str]:
+    """Gather all of the requirements from provider manifests."""
+    dependencies: list[str] = []
+    providers_path = "music_assistant/server/providers"
+    for dir_str in os.listdir(providers_path):
+        dir_path = os.path.join(providers_path, dir_str)
+        if not os.path.isdir(dir_path):
+            continue
+        # get files in subdirectory
+        for file_str in os.listdir(dir_path):
+            file_path = os.path.join(dir_path, file_str)
+            if not os.path.isfile(file_path):
+                continue
+            if file_str != "manifest.json":
+                continue
+
+            with open(file_path) as _file:
+                provider_manifest = json.loads(_file.read())
+                dependencies += provider_manifest["requirements"]
+    return dependencies
+
+
+def main() -> int:
+    """Run the script."""
+    if not os.path.isfile("requirements_all.txt"):
+        print("Run this from MA root dir")
+        return 1
+
+    core_reqs = gather_core_requirements()
+    extra_reqs = gather_requirements_from_manifests()
+
+    # use intermediate dict to detect duplicates
+    # TODO: compare versions and only store most recent
+    final_requirements: dict[str, str] = {}
+    for req_str in core_reqs + extra_reqs:
+        if match := PACKAGE_REGEX.search(req_str):
+            package_name = match.group(1).lower().replace("_", "-")
+        elif package_name in final_requirements:
+            # duplicate package without version is safe to ignore
+            continue
+        else:
+            print("Found requirement without version specifier: %s" % req_str)
+            package_name = req_str
+
+        existing = final_requirements.get(package_name)
+        if existing:
+            print("WARNING: ignore duplicate package: %s - existing: %s" % package_name, existing)
+            continue
+        final_requirements[package_name] = req_str
+
+    content = "# WARNING: this file is autogenerated!\n\n"
+    for req_key in sorted(final_requirements):
+        req_str = final_requirements[req_key]
+        content += f"{req_str}\n"
+    Path("requirements_all.txt").write_text(content)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/script/run-in-env.sh b/script/run-in-env.sh
new file mode 100755 (executable)
index 0000000..271e7a4
--- /dev/null
@@ -0,0 +1,22 @@
+#!/usr/bin/env sh
+set -eu
+
+# Activate pyenv and virtualenv if present, then run the specified command
+
+# pyenv, pyenv-virtualenv
+if [ -s .python-version ]; then
+    PYENV_VERSION=$(head -n 1 .python-version)
+    export PYENV_VERSION
+fi
+
+# other common virtualenvs
+my_path=$(git rev-parse --show-toplevel)
+
+for venv in venv .venv .; do
+  if [ -f "${my_path}/${venv}/bin/activate" ]; then
+    . "${my_path}/${venv}/bin/activate"
+    break
+  fi
+done
+
+exec "$@"