From f0a2aa5036fce28535901ec3827162313b3f9a16 Mon Sep 17 00:00:00 2001 From: Roman Nebaluev Date: Wed, 13 Nov 2024 18:28:06 +0500 Subject: [PATCH] change download method, add lossless quality --- yandex_fuse/virt_fs.py | 2 +- yandex_fuse/ya_music_fs.py | 76 +++++++----- yandex_fuse/ya_player.py | 241 ++++++++++++++++++++----------------- 3 files changed, 178 insertions(+), 141 deletions(-) diff --git a/yandex_fuse/virt_fs.py b/yandex_fuse/virt_fs.py index d13264d..128df25 100644 --- a/yandex_fuse/virt_fs.py +++ b/yandex_fuse/virt_fs.py @@ -95,7 +95,7 @@ def insert(self) -> tuple[str, dict[str, Any]]: INSERT INTO {self.__tablename__} ({columns}) VALUES - '({placeholders})' + ({placeholders}) """ return query, data diff --git a/yandex_fuse/ya_music_fs.py b/yandex_fuse/ya_music_fs.py index 6f3ba63..de79505 100644 --- a/yandex_fuse/ya_music_fs.py +++ b/yandex_fuse/ya_music_fs.py @@ -149,8 +149,11 @@ async def read_from(self, offset: int, size: int) -> bytes: self.__download_task.result() self.__ready_read.clear() - with suppress(AsyncTimeoutError): + try: await wait_for(self.__ready_read.wait(), timeout=5) + except AsyncTimeoutError: + log.warning("Slow downloading %s", self.__track.name) + self.__total_read += size return bytes(self.__bytes.getbuffer()[offset : offset + size]) @@ -190,7 +193,7 @@ async def download(self) -> None: continue buffer.write(chunk) - new_buffer = self.__tag.to_bytes(buffer) + new_buffer = self.__tag.to_bytes(buffer, self.__track.codec) if new_buffer is None: continue self._write_tag( @@ -214,6 +217,7 @@ async def download(self) -> None: raise else: log.debug("Track %s downloaded", self.__track.name) + self.__ready_read.set() if self.__download_task is None: return download_task = self.__download_task @@ -230,6 +234,7 @@ class SQLTrack(SQLRow): playlist_id: str codec: str bitrate: int + quality: str size: int artist: str title: str @@ -291,6 +296,7 @@ class YaMusicFS(VirtFS): REFERENCES playlists(playlist_id) ON DELETE RESTRICT, codec BLOB(8) NOT NULL, bitrate INT NOT NULL, + quality TEXT(20) NOT NULL, size INT NOT NULL, artist TEXT(255), title TEXT(255), @@ -391,27 +397,44 @@ async def get_or_update_direct_link( except ClientError as err: log.error("Fail get direct link: %r", err) # noqa: TRY400 - new_direct_link = await self._ya_player.get_download_link( + new_direct_links = await self._ya_player.get_download_links( track_id, codec, bitrate_in_kbps, ) - if new_direct_link is None: + + if new_direct_links is None: return None - expired = int((time.time() + 8600) * 1e9) - cursor.execute( - """ - INSERT INTO direct_link - (track_id, link, expired) - VALUES(?, ?, ?) - ON CONFLICT(track_id) - DO UPDATE SET link=excluded.link,expired=excluded.expired - """, - (track_id, new_direct_link, expired), - ) - log.debug("Direct link: %s, track: %s", new_direct_link, track_id) - return new_direct_link + for new_direct_link in new_direct_links: + log.debug("Check direct link %s", new_direct_link) + try: + async with self._client_session.request( + "HEAD", + new_direct_link, + ) as resp: + if not resp.ok: + continue + except ClientError as err: + log.error("Fail get direct link: %r", err) # noqa: TRY400 + continue + + expired = int((time.time() + 8600) * 1e9) + cursor.execute( + """ + INSERT INTO direct_link + (track_id, link, expired) + VALUES(?, ?, ?) + ON CONFLICT(track_id) + DO UPDATE SET link=excluded.link,expired=excluded.expired + """, + (track_id, new_direct_link, expired), + ) + log.debug( + "Direct link: %s, track: %s", new_direct_link, track_id + ) + return new_direct_link + return None def _get_playlist_by_id(self, playlist_id: str) -> SQLPlaylist | None: return SQLPlaylist.from_row( @@ -569,7 +592,7 @@ async def _update_track( byte = BytesIO() async for chunk in resp.content.iter_chunked(1024): byte.write(chunk) - new_buffer = track.tag.to_bytes(byte) + new_buffer = track.tag.to_bytes(byte, track.codec) if new_buffer is None: continue track.size += track.tag.size @@ -628,6 +651,7 @@ async def _update_track( playlist_id=playlist_id, codec=track.codec, bitrate=track.bitrate_in_kbps, + quality=track.quality, size=track.size, artist=tag["artist"], title=tag["title"], @@ -974,7 +998,7 @@ async def open( buffer = await self._get_buffer(track) if buffer is None: - raise FUSEError(errno.EPERM) + raise FUSEError(errno.EPIPE) file_info = await super().open(inode, flags, ctx) if flags & os.O_RDWR or flags & os.O_WRONLY: @@ -1015,7 +1039,7 @@ async def read(self, fd: FileHandleT, offset: int, size: int) -> bytes: stream_reader.track.playlist_id ) - if playlist is not None and playlist.batch_id is not None: + if playlist is not None and playlist.batch_id: await self._ya_player.feedback_track( stream_reader.track.track_id, "trackStarted", @@ -1050,7 +1074,7 @@ async def release(self, fd: FileHandleT) -> None: stream_reader.track.playlist_id ) - if playlist is not None and playlist.batch_id is not None: + if playlist is not None and playlist.batch_id: await self._ya_player.feedback_track( stream_reader.track.track_id, "trackFinished", @@ -1098,18 +1122,14 @@ async def setxattr( def xattrs(self, inode: InodeT) -> dict[str, Any]: return { "inode": inode, - "nlookup": len(self._nlookup), - "inode_map_fd": self._inode_map_fd, - "queue_invalidate_inode": self.queue_later_invalidate_inode, + "inode_map_fd": self._inode_map_fd.get(inode), "stream": { fd: { - "name": stream.track.name, + "name": stream.track.name.decode(), "size": stream.track.size, "codec": stream.track.codec, "bitrate": stream.track.bitrate, - "play_second": stream.buffer.total_second() - if stream.buffer is not None - else 0, + "play_second": stream.buffer.total_second(), } for fd, stream in self._fd_map_stream.items() }, diff --git a/yandex_fuse/ya_player.py b/yandex_fuse/ya_player.py index 9ffaf2d..ee928e0 100644 --- a/yandex_fuse/ya_player.py +++ b/yandex_fuse/ya_player.py @@ -1,6 +1,8 @@ from __future__ import annotations import base64 +import hashlib +import hmac import json import logging import struct @@ -12,6 +14,8 @@ from io import BytesIO from typing import TYPE_CHECKING, Any, ClassVar +from mutagen import MutagenError # type: ignore[attr-defined] +from mutagen.flac import FLAC, FLACNoHeaderError from mutagen.id3 import ( # type: ignore[attr-defined] TALB, TCON, @@ -24,11 +28,13 @@ from mutagen.mp4 import MP4 from yandex_music import ( # type: ignore[import-untyped] ClientAsync, - DownloadInfo, Track, YandexMusicObject, ) from yandex_music.utils import model # type: ignore[import-untyped] +from yandex_music.utils.sign_request import ( # type: ignore[import-untyped] + DEFAULT_SIGN_KEY, +) from yandex_fuse.request import YandexClientRequest @@ -88,15 +94,15 @@ def _to_mp4_tag(self, stream: BytesIO) -> bytes | None: return None audiofile = MP4(fileobj=tag_stream) # type: ignore[no-untyped-call] - # audiofile.delete(fileobj=tag_stream) - # audiofile.pop("----:com.apple.iTunes:iTunSMPB", None) + audiofile.pop("----:com.apple.iTunes:iTunSMPB", None) # type: ignore[no-untyped-call] + # https://mutagen.readthedocs.io/en/latest/api/mp4.html#mutagen.mp4.MP4Tags audiofile["\xa9nam"] = self.title audiofile["\xa9alb"] = self.album audiofile["\xa9ART"] = self.artist audiofile["\xa9day"] = self.year audiofile["\xa9gen"] = self.genre - # audiofile["covr"] = + stream.seek(0) audiofile.save(fileobj=tag_stream) # type: ignore[no-untyped-call] @@ -140,26 +146,52 @@ def _to_mp3_tag(self, stream: BytesIO) -> bytes | None: audiofile.save(fileobj=new_stream) - self.size = len(stream.getbuffer()) - len(new_stream.getbuffer()) + self.size = len(new_stream.getbuffer()) - len(stream.getbuffer()) + return bytes(new_stream.getbuffer()) + + def _to_flac_tag(self, stream: BytesIO) -> bytes | None: + new_stream = BytesIO() + new_stream.write(stream.read()) + new_stream.seek(0) + try: + audiofile = FLAC(fileobj=new_stream) + except (FLACNoHeaderError, MutagenError): + return None + # https://exiftool.org/TagNames/Vorbis.html + + audiofile["TITLE"] = self.title + audiofile["ALBUM"] = self.album + audiofile["ARTIST"] = self.artist + audiofile["DATE"] = self.year + audiofile["GENRE"] = self.genre + new_stream.seek(0) + + audiofile.save(fileobj=new_stream) + self.size = len(new_stream.getbuffer()) - len(stream.getbuffer()) return bytes(new_stream.getbuffer()) - def to_bytes(self, stream: BytesIO) -> bytes | None: + def to_bytes(self, stream: BytesIO, codec: str) -> bytes | None: buffer = bytearray(stream.getbuffer()) current_offset = stream.tell() stream.seek(0) try: - if len(buffer) <= MP3_HEADER_MIN_SIZE: - return None - if MP3.score("", None, buffer): # type: ignore[no-untyped-call] - return self._to_mp3_tag(stream) - - if len(buffer) <= MP4_HEADER_MIN_SIZE: - return None - if MP4.score(None, None, buffer): # type: ignore[no-untyped-call] - return self._to_mp4_tag(stream) - + if codec == "flac" and FLAC.score(".flac", None, buffer): # type: ignore[no-untyped-call] + return self._to_flac_tag(stream) + + if codec == "mp3": + if len(buffer) <= MP3_HEADER_MIN_SIZE: + return None + if MP3.score("", None, buffer): # type: ignore[no-untyped-call] + return self._to_mp3_tag(stream) + elif codec == "aac": + if len(buffer) <= MP4_HEADER_MIN_SIZE: + return None + if MP4.score(None, None, buffer): # type: ignore[no-untyped-call] + return self._to_mp4_tag(stream) + else: + pass finally: stream.seek(current_offset) return bytes(stream.getbuffer()) @@ -172,6 +204,7 @@ class ExtendTrack(Track): # type: ignore[misc] size: int = 0 codec: str = "" bitrate_in_kbps: int = 0 + quality: str = "" direct_link: str = "" _tag: TrackTag | None = None @@ -190,15 +223,13 @@ def from_track(cls, track: Track) -> ExtendTrack: def save_name(self) -> str: artists = ", ".join(self.artists_name()) name = f"{artists} - {self.title}" + + codec2ext_name = {"flac": "flac", "aac": "m4a", "mp3": "mp3"} + if self.version: name += f" ({self.version})" - if self.codec == "aac": - ext_name = "m4a" - elif self.codec == "mp3": - ext_name = "mp3" - else: - ext_name = "unknown" + ext_name = codec2ext_name.get(self.codec, "unknown") return f"{name}.{ext_name}".replace("/", "-") async def _download_image(self) -> str: @@ -235,68 +266,19 @@ def tag(self) -> TrackTag: return self._tag - def _choose_best_dowanload_info(self) -> DownloadInfo: - best_bitrate_in_kbps = {"aac": 0, "mp3": 0} - track_info: dict[str, DownloadInfo | None] = {"aac": None, "mp3": None} - - best_codec = self.client.settings["best_codec"].lower() - self.download_info: list[DownloadInfo] | None - if self.download_info is None: - msg = ( - f"Download info for track {self.title} empty!" - "Call get_download_info(async) before get info." - ) - raise ValueError( - msg, - ) - - for info in self.download_info: - log.debug( - "Track %s %s/%d", - self.title, - info.codec, - info.bitrate_in_kbps, - ) - - best_bitrate_in_kbps[info.codec] = max( - info.bitrate_in_kbps, - best_bitrate_in_kbps[info.codec], - ) - - if info.bitrate_in_kbps >= best_bitrate_in_kbps[info.codec]: - track_info[info.codec] = info - - track_codec_info = track_info.pop(best_codec) - if track_codec_info is None: - _, track_codec_info = track_info.popitem() - if track_codec_info is None: - raise RuntimeError(f"Track info {self.title} is empty.") - - log.warning( - "Best codec %s from track %s not available. Fallback. %s", - best_codec, - self.title, - track_codec_info.codec, - ) - - log.debug( - "Track %s choose codec %s/%d", - self.title, - track_codec_info.codec, - track_codec_info.bitrate_in_kbps, - ) - - return track_codec_info - - async def update_download_info(self) -> None: - if self.download_info is None: - log.debug('Track "%s" update download info.', self.title) - await self.get_download_info_async() - download_info = self._choose_best_dowanload_info() - self.codec = download_info.codec - self.bitrate_in_kbps = download_info.bitrate_in_kbps - self.download_info = [download_info] +@model +class DownloadInfo(YandexMusicObject): # type: ignore[misc] + quality: str + codec: str + urls: list[str] + url: str + bitrate: int + track_id: str + size: int + transport: str + gain: bool + real_id: str class YandexMusicPlayer(ClientAsync): # type: ignore[misc] @@ -305,7 +287,7 @@ class YandexMusicPlayer(ClientAsync): # type: ignore[misc] "last_track": None, "from_id": f"music-{uuid.uuid4()}", "station_id": "user:onyourwave", - "best_codec": "aac", + "quality": "lossless", "blacklist": [], } @@ -329,7 +311,6 @@ def __init__( self.__settings = self._default_settings self.save_settings() - self.__client_session = client_session super().__init__( self.__settings["token"], request=YandexClientRequest(client_session), @@ -389,7 +370,8 @@ async def load_tracks( if str(track.id) in exclude_track_ids: continue extend_track = ExtendTrack.from_track(track) - await extend_track.update_download_info() + await self._choose_best_dowanload_info(extend_track) + yield extend_track async def next_tracks( @@ -466,7 +448,7 @@ async def next_tracks( if extend_track.track_id in exclude_track_ids: continue - await extend_track.update_download_info() + await self._choose_best_dowanload_info(extend_track) tracks.add(extend_track.save_name) yield extend_track @@ -477,37 +459,63 @@ async def next_tracks( def get_last_station_info(self) -> tuple[str, str]: return self.__last_station_id - async def get_download_link( + async def _get_download_info(self, track_id: str) -> DownloadInfo: + # https://github.com/MarshalX/yandex-music-api/issues/656#issuecomment-2466722441 + timestamp = int(time.time()) + params = { + "ts": timestamp, + "trackId": track_id, + "quality": self.__settings["quality"], + "codecs": "flac,aac,he-aac,mp3", + "transports": "raw", + } + res = "".join(str(e) for e in params.values()).replace(",", "") + hmac_sign = hmac.new( + DEFAULT_SIGN_KEY.encode(), + res.encode(), + hashlib.sha256, + ) + sign = base64.b64encode(hmac_sign.digest()).decode()[:-1] + params["sign"] = sign + + resp: dict[str, Any] = await self._request.get( + "https://api.music.yandex.net/get-file-info", params=params + ) + return DownloadInfo(**resp["download_info"]) + + async def _choose_best_dowanload_info(self, track: ExtendTrack) -> None: + download_info = await self._get_download_info(track.id) + + track.codec = download_info.codec + track.bitrate_in_kbps = download_info.bitrate + track.quality = download_info.quality + + async def get_download_links( self, track_id: str, codec: str, bitrate_in_kbps: int, - ) -> str | None: - download_info = await self.tracks_download_info(track_id) - best_bitrate_in_kbps = {"aac": 0, "mp3": 0} - track_info: dict[str, DownloadInfo | None] = {"aac": None, "mp3": None} - - for info in download_info: - best_bitrate_in_kbps[info.codec] = max( - info.bitrate_in_kbps, - best_bitrate_in_kbps[info.codec], + ) -> list[str] | None: + download_info = await self._get_download_info(track_id) + log.debug("Track %s, download info: %r", track_id, download_info) + if bitrate_in_kbps != download_info.bitrate: + log.warning( + "Track %s, bitrate not match: %d != %d", + track_id, + bitrate_in_kbps, + download_info.bitrate, ) + return None + if codec != download_info.codec: + log.warning( + "Track %s, codec not match: %d != %d", + track_id, + codec, + download_info.codec, + ) + return None - if info.bitrate_in_kbps >= best_bitrate_in_kbps[info.codec]: - track_info[info.codec] = info - - if info.codec == codec and info.bitrate_in_kbps == bitrate_in_kbps: - direct_link: str = await info.get_direct_link_async() - return direct_link - - log.warning( - "Track %s, codec: %s, bitrate kbps: %d. Not match!", - track_id, - codec, - bitrate_in_kbps, - ) - - return None + return download_info.urls async def feedback_track( self, @@ -518,6 +526,15 @@ async def feedback_track( total_played_seconds: int, ) -> None: # trackStarted, trackFinished, skip. + log.debug( + "Feedback send, track id: %s, feedback: %s, " + "station id: %s, batch_id: %s, seconds %d", + track_id, + feedback, + station_id, + batch_id, + total_played_seconds, + ) try: await self.rotor_station_feedback( station=station_id,