From: Marcel van der Veldt Date: Thu, 9 Mar 2023 14:00:05 +0000 (+0100) Subject: build wheels in docker X-Git-Url: https://git.kitaultman.com/?a=commitdiff_plain;h=c860d510fb1809c51efcce600741031757b0117a;p=music-assistant-server.git build wheels in docker --- diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 75ab08e2..7a7414c7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -59,13 +59,20 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Log in to the Github Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.API_TOKEN_GITHUB }} + - name: Build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} context: . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ee2805a..5e064d2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,9 +30,18 @@ repos: additional_dependencies: - tomli - # - repo: https://github.com/pre-commit/mirrors-mypy - # rev: v0.990 - # hooks: - # - id: mypy - # additional_dependencies: [types-all] - # exclude: ^examples/ + - repo: local + hooks: + # - id: pylint + # name: pylint + # entry: script/run-in-env.sh pylint -j 0 --ignore-missing-annotations=y + # language: script + # types: [python] + # files: ^music_assistant/.+\.py$ + - id: gen_requirements_all + name: gen_requirements_all + entry: script/run-in-env.sh python3 -m script.gen_requirements_all + pass_filenames: false + language: script + types: [text] + files: ^(music_assistant/.+/manifest\.json|pyproject\.toml|\.pre-commit-config\.yaml|script/gen_requirements_all\.py)$ diff --git a/Dockerfile b/Dockerfile index 5e6b07dc..ece48b72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,55 +8,52 @@ ARG PYTHON_VERSION="3.11" # Build Wheels # # # ##################################################################### -FROM python:${PYTHON_VERSION}-slim as wheels-builder +FROM python:${PYTHON_VERSION}-alpine as wheels-builder ARG HASS_ARCH -ENV PIP_EXTRA_INDEX_URL=https://www.piwheels.org/simple ENV PATH="${PATH}:/root/.cargo/bin" # Install buildtime packages RUN set -x \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - gcc \ + && apk add --no-cache \ + alpine-sdk \ + patchelf \ + build-base \ + cmake \ git \ + linux-headers \ + autoconf \ + automake \ + cargo \ + libffi \ libffi-dev \ - libssl-dev - -RUN set -x \ - \ - && if [ ${HASS_ARCH} = "amd64" ]; then RUST_ARCH="x86_64-unknown-linux-gnu"; fi \ - && if [ ${HASS_ARCH} = "armv7" ]; then RUST_ARCH="armv7-unknown-linux-gnueabihf"; fi \ - && if [ ${HASS_ARCH} = "aarch64" ]; then RUST_ARCH="aarch64-unknown-linux-gnu"; fi \ - \ - && curl -o rustup-init https://static.rust-lang.org/rustup/dist/${RUST_ARCH}/rustup-init \ - && chmod +x rustup-init \ - && ./rustup-init -y --no-modify-path --profile minimal --default-host ${RUST_ARCH} + git WORKDIR /wheels -COPY requirements.txt . +COPY requirements_all.txt . -# build python wheels +# build python wheels for all dependencies RUN set -x \ - && pip wheel -r .[server] + && pip install --upgrade pip \ + && pip install build \ + && pip wheel -r requirements_all.txt --find-links "https://wheels.home-assistant.io/musllinux/" +# build music assistant wheel +COPY music_assistant music_assistant +COPY pyproject.toml . +COPY MANIFEST.in . +RUN python3 -m build --wheel --outdir /wheels --skip-dependency-check ##################################################################### # # # Final Image # # # ##################################################################### -FROM python:${PYTHON_VERSION}-slim AS final-build +FROM python:${PYTHON_VERSION}-alpine AS final-build WORKDIR /app -ENV DEBIAN_FRONTEND="noninteractive" - RUN set -x \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ + && apk add --no-cache \ ca-certificates \ curl \ git \ @@ -75,7 +72,8 @@ RUN set -x \ # Install all built wheels RUN --mount=type=bind,target=/tmp/wheels,source=/wheels,from=wheels-builder,rw \ set -x \ - && pip install --no-cache-dir -f /tmp/wheels -r /tmp/wheels/requirements.txt + && pip install --upgrade pip \ + && pip install --no-cache-dir /tmp/wheels/*.whl # Required to persist build arg ARG BUILD_VERSION diff --git a/README.md b/README.md index 692dc720..dfd56f76 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,7 @@ Music Assistant [![pypi_badge](https://img.shields.io/pypi/v/music_assistant.svg)](https://pypi.python.org/pypi/music_assistant) -**Music Assistant** +**Music Assistant Server** -Python library for managing online and offline music sources and orchestrate playback to devices. - -This library has been created mainly for the [Home Assistant Integration](https://github.com/music-assistant/hass-music-assistant) but can be used stand-alone as well in other projects. -See the examples folder for some simple code samples or have a look at the [Home Assistant Integration](https://github.com/music-assistant/hass-music-assistant) for discovering the full potential. +Music Assistant is a free, opensource Media library manager that connects to your streaming services and a wide range of connected speakers. The server is the beating heart, the core of Music Assistant and must run on an always-on device like a Raspberry Pi, a NAS or an Intel NUC or alike. diff --git a/requirements_all.txt b/requirements_all.txt new file mode 100644 index 00000000..2e35199b --- /dev/null +++ b/requirements_all.txt @@ -0,0 +1,35 @@ +# WARNING: this file is autogenerated! + +aiofiles==23.1.0 +aiohttp==3.8.4 +aiorun==2022.11.1 +aioslimproto==2.2.0 +aiosqlite==0.18.0 +async-upnp-client==0.33.1 +asyncio-throttle==1.0.2 +black==23.1.0 +codespell==2.2.2 +cryptography==39.0.2 +databases==0.7.0 +getmac==0.8.2 +mashumaro==3.5 +memory-tempfile==2.2.3 +music-assistant-frontend==20230308.0 +mypy==1.0.1 +orjson==3.8.6 +pillow==9.4.0 +pre-commit==2.20.0 +PyChromecast==13.0.4 +pysmb==1.2.9.1 +pytest==7.2.1 +pytest-aiohttp==1.0.4 +pytest-cov==4.0.0 +python-slugify==7.0.0 +pytube==12.1.2 +ruff==0.0.254 +shortuuid==1.0.11 +soco==0.29.1 +unidecode==1.3.6 +xmltodict==0.13.0 +ytmusicapi==0.25.0 +zeroconf==0.47.3 diff --git a/script/gen_requirements_all.py b/script/gen_requirements_all.py new file mode 100644 index 00000000..c696ee73 --- /dev/null +++ b/script/gen_requirements_all.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Generate updated constraint and requirements files.""" +from __future__ import annotations + +import json +import os +import re +import sys +import tomllib +from pathlib import Path + +PACKAGE_REGEX = re.compile(r"^(?:--.+\s)?([-_\.\w\d]+).*==.+$") + + +def gather_core_requirements() -> list[str]: + """Gather core requirements out of pyproject.toml.""" + with open("pyproject.toml", "rb") as fp: + data = tomllib.load(fp) + dependencies: list[str] = data["project"]["optional-dependencies"]["server"] + dependencies += data["project"]["dependencies"] + dependencies += data["project"]["optional-dependencies"]["test"] + return dependencies + + +def gather_requirements_from_manifests() -> list[str]: + """Gather all of the requirements from provider manifests.""" + dependencies: list[str] = [] + providers_path = "music_assistant/server/providers" + for dir_str in os.listdir(providers_path): + dir_path = os.path.join(providers_path, dir_str) + if not os.path.isdir(dir_path): + continue + # get files in subdirectory + for file_str in os.listdir(dir_path): + file_path = os.path.join(dir_path, file_str) + if not os.path.isfile(file_path): + continue + if file_str != "manifest.json": + continue + + with open(file_path) as _file: + provider_manifest = json.loads(_file.read()) + dependencies += provider_manifest["requirements"] + return dependencies + + +def main() -> int: + """Run the script.""" + if not os.path.isfile("requirements_all.txt"): + print("Run this from MA root dir") + return 1 + + core_reqs = gather_core_requirements() + extra_reqs = gather_requirements_from_manifests() + + # use intermediate dict to detect duplicates + # TODO: compare versions and only store most recent + final_requirements: dict[str, str] = {} + for req_str in core_reqs + extra_reqs: + if match := PACKAGE_REGEX.search(req_str): + package_name = match.group(1).lower().replace("_", "-") + elif package_name in final_requirements: + # duplicate package without version is safe to ignore + continue + else: + print("Found requirement without version specifier: %s" % req_str) + package_name = req_str + + existing = final_requirements.get(package_name) + if existing: + print("WARNING: ignore duplicate package: %s - existing: %s" % package_name, existing) + continue + final_requirements[package_name] = req_str + + content = "# WARNING: this file is autogenerated!\n\n" + for req_key in sorted(final_requirements): + req_str = final_requirements[req_key] + content += f"{req_str}\n" + Path("requirements_all.txt").write_text(content) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/run-in-env.sh b/script/run-in-env.sh new file mode 100755 index 00000000..271e7a4a --- /dev/null +++ b/script/run-in-env.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +set -eu + +# Activate pyenv and virtualenv if present, then run the specified command + +# pyenv, pyenv-virtualenv +if [ -s .python-version ]; then + PYENV_VERSION=$(head -n 1 .python-version) + export PYENV_VERSION +fi + +# other common virtualenvs +my_path=$(git rev-parse --show-toplevel) + +for venv in venv .venv .; do + if [ -f "${my_path}/${venv}/bin/activate" ]; then + . "${my_path}/${venv}/bin/activate" + break + fi +done + +exec "$@"