Update github actions to new python version, add macOS to runners. Al…

…so make the tests working again

.github/workflows/tests.yml

@@ -9,8 +9,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest, windows-latest ]
-        python-version: [ '3.8', '3.9', '3.10' ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: [ '3.10', '3.11' ]
 
     steps:
       - uses: actions/checkout@v2

pyproject.toml

@@ -3,8 +3,9 @@ requires = ["setuptools>=42.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.mypy]
-check_untyped_defs = true
+strict = true
 disallow_any_generics = true
+check_untyped_defs = true
 ignore_missing_imports = true
 show_error_codes = true
 strict_equality = true
@@ -14,4 +15,4 @@ warn_unreachable = true
 warn_unused_configs = true
 no_implicit_reexport = true
 disallow_untyped_defs = true
-strict = true
+

requirements_dev.txt

@@ -1,5 +1,6 @@
 pytest~=7.4.0
 pytest-cov~=4.1.0
+pytest-asyncio~=0.21.1
 mypy~=1.5.0
 flake8~=6.1.0
 tox~=4.8.0

setup.cfg

@@ -47,6 +47,7 @@ console_scripts =
 testing =
     pytest~=7.4.0
     pytest-cov~=4.1.0
+    pytest-asyncio~=0.21.1
     mypy~=1.5.0
     flake8~=6.1.0
     tox~=4.8.0

src/isisdl/__main__.py

@@ -3,19 +3,18 @@
 import sys
 import time
 from asyncio import create_task
-from threading import Thread
 
 import isisdl.compress as compress
 from isisdl.api.crud import authenticate_new_session
 from isisdl.api.endpoints import CourseContentsAPI, UserCourseListAPI
 from isisdl.api.rate_limiter import RateLimiter, ThrottleType
 from isisdl.backend import sync_database
 from isisdl.backend.config import init_wizard, config_wizard
-from isisdl.backend.crud import read_config, read_user, create_default_config, store_user
+from isisdl.backend.crud import read_config, read_user
 from isisdl.backend.request_helper import CourseDownloader
 from isisdl.db_conf import init_database, DatabaseSessionMaker
-from isisdl.settings import is_first_time, is_static, forbidden_chars, has_ffmpeg, fstype, is_windows, working_dir_location, python_executable, is_macos, is_online, DEBUG_ASSERTS
-from isisdl.utils import args, acquire_file_lock_or_exit, generate_error_message, install_latest_version, export_config, database_helper, config, migrate_database, Config, compare_download_diff
+from isisdl.settings import is_first_time, is_static, forbidden_chars, has_ffmpeg, fstype, is_windows, working_dir_location, python_executable, is_macos, is_online
+from isisdl.utils import args, acquire_file_lock_or_exit, generate_error_message, install_latest_version, export_config, database_helper, config, migrate_database, Config, compare_download_diff, HumanBytes
 from isisdl.version import __version__
 
 
@@ -39,7 +38,8 @@ async def getter(id: int, limiter: RateLimiter) -> None:
     while True:
         token = await limiter.get(ThrottleType.free_for_all)
         await asyncio.sleep(0.01)
-        print(f"Got token from task {id}!")
+        print(f"Got token from task {id} and I can download {token.num_bytes} bytes! That is {HumanBytes.format(token.num_bytes)}")
+
 
 async def _new_main() -> None:
     with DatabaseSessionMaker() as db:
@@ -58,14 +58,14 @@ async def _new_main() -> None:
 
         s = time.perf_counter()
         contents = await CourseContentsAPI.get(db, session, courses)
+        _ = contents
         print(f"{time.perf_counter() - s:.3f}s")
 
         limiter = RateLimiter(20)
         create_task(getter(1, limiter))
         create_task(getter(2, limiter))
         create_task(getter(3, limiter))
 
-
         # TODO: How to deal with crashing threads
         #   - Have a menu which enables 3 choices:
         #     - restart with same file
@@ -79,7 +79,6 @@ async def _new_main() -> None:
 
 
 def _main() -> None:
-
     init_database()
 
     if is_first_time:
@@ -137,8 +136,6 @@ def _main() -> None:
 
     asyncio.run(_new_main())
 
-    return
-
     install_latest_version()
 
     if args.update:

src/isisdl/api/crud.py

@@ -44,7 +44,7 @@ async def authenticate_new_session(user: User, config: Config) -> AuthenticatedS
             return None
 
         # Extract the session key
-        text =  await response.text()
+        text = await response.text()
         _session_key = re.search(r"\"sesskey\":\"(.*?)\"", text)
         if _session_key is None:
             return None

src/isisdl/api/endpoints.py

@@ -3,7 +3,8 @@
 import asyncio
 from collections import defaultdict
 from json import JSONDecodeError
-from typing import Any, Self, cast
+from typing import Any, cast
+from typing_extensions import Self
 
 from sqlalchemy.orm import Session as DatabaseSession
 
@@ -44,7 +45,7 @@ async def _get(cls, session: AuthenticatedSession, data: dict[str, Any] | None =
                 return None
 
             try:
-                match (x := await response.json()):
+                match await response.json():
                     case {"errorcode": _} | {"exception": _}:
                         return None
 

src/isisdl/api/rate_limiter.py

@@ -5,8 +5,7 @@
 from asyncio import get_event_loop, create_task, Condition, Task, Event, wait_for
 from dataclasses import dataclass, field
 from enum import Enum
-from numbers import Number
-from typing import TYPE_CHECKING, MutableMapping, overload
+from typing import TYPE_CHECKING
 
 from typing_extensions import Self
 
@@ -118,7 +117,7 @@ def __init__(self, num_tokens_per_iteration: int | None, _condition: Condition |
 
     @classmethod
     def from_bandwidth(cls, num_mbits: float, _condition: Condition | None = None) -> Self:
-        return cls(num_mbits * 1024 ** 2 // download_chunk_size * token_queue_refresh_rate)
+        return cls(int(num_mbits * 1024 ** 2 / download_chunk_size * token_queue_refresh_rate))
 
     def calculate_max_num_tokens(self) -> int:
         if self.rate is None:
@@ -254,4 +253,4 @@ async def _get(self, media_type: ThrottleType, block: bool = True) -> Token | No
 
     async def used_bandwidth(self) -> float:
         async with self.refill_condition:
-            return sum(self.bytes_downloaded) / token_queue_bandwidths_save_for
+            return sum(self.bytes_downloaded) / token_queue_bandwidths_save_for

src/isisdl/backend/database_helper.py

@@ -46,7 +46,7 @@ class DatabaseHelper:
     con: Connection
     cur: Cursor
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     lock = Lock()
     _bad_urls: dict[str, BadUrl] = dict()

src/isisdl/backend/request_helper.py

@@ -39,7 +39,7 @@ class SessionWithKey(Session):
     key: str
     token: str
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     _lock = Lock()
 
@@ -197,7 +197,7 @@ class PreMediaContainer:
     is_cached: bool
     parent_path: Path
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     def __init__(self, url: str, course: Course, media_type: MediaType, name: str | None = None, relative_location: str | None = None, size: MediaContainerSize = MediaContainerSize.no_size.new(),
                  time: int | None = None):
@@ -241,7 +241,7 @@ class MediaContainer:
     _newly_downloaded: bool
     _newly_discovered: bool
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     def __init__(self, _name: str, url: str, download_url: str, time: int | None, course: Course,
                  media_type: MediaType, size: MediaContainerSize, checksum: Optional[str] = None,
@@ -596,7 +596,7 @@ class Course:
     name: str
     course_id: int
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     def __init__(self, displayname: str, _name: str, name: str, course_id: int) -> None:
         self.displayname = displayname

src/isisdl/compress.py

@@ -154,7 +154,7 @@ class CompressStatus(Thread):
     last_text_len: int
     last_file_size_stat: int
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     def __init__(self, files: List[MediaContainer], helper: RequestHelper) -> None:
         self.files = files

src/isisdl/settings.py

@@ -300,14 +300,14 @@ def error_exit(code: int, reason: str) -> NoReturn:
 # @formatter:off
 extern_ignore = re.compile(
     "(?:"
-        "(?:https://)?(:?"
+        "(?:https://)?(:?"  # noqa:E131
         # Full urls
         "berlin.de|tu-berlin.de|archive.org|b.sc|m.sc|nebula.stream"
         # Spam urls
         "|zmeu.us|69n.de|4-5.FM|6cin.de|6e.de|6e.de|6e.de|9.FM|10.FM|s.th|lin.de|flinga.fi|ICUnet.AG"
     "))"
         # Python files
-        "|\w+\.py"
+        r"|\w+\.py"
     "|"
         # Match part of url's
         ".*(?:"

src/isisdl/utils.py

@@ -19,7 +19,7 @@
 import sys
 import time
 import traceback
-from asyncio import BaseEventLoop, get_event_loop, AbstractEventLoop
+from asyncio import get_event_loop, AbstractEventLoop
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import ExitStack
@@ -128,7 +128,7 @@ class Config:
 
     auto_subscribed_courses: Optional[List[int]]
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     default_config: Dict[str, Union[bool, str, int, None, Dict[int, str]]] = {
         "password_encrypted": False,
@@ -1085,7 +1085,7 @@ class User:
     username: str
     password: str
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     def __init__(self, username: str, password: str):
         self.username = username
@@ -1336,7 +1336,7 @@ class DataLogger(Thread):
     done: Queue[None]  # Used to communicate
     generic_msg: dict[str, Any]
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     def __init__(self) -> None:
         self.s = Session()
@@ -1413,7 +1413,7 @@ class DownloadThrottler(Thread):
     download_rate: int
     refresh_rate: float
 
-    __slots__ = tuple(__annotations__)  # type: ignore
+    __slots__ = tuple(__annotations__)
 
     token = Token()
     timestamps: list[float] = []
@@ -1545,6 +1545,7 @@ def normalize(it: dict[T, int]) -> dict[T, float]:
     total = sum(it.values())
     return {k: v / total if not isclose(total, 0) else 0 for k, v in it.items()}
 
+
 def get_async_time(event_loop: AbstractEventLoop | None = None) -> float:
     return (event_loop or get_event_loop()).time()
 
@@ -1571,10 +1572,9 @@ def format(num: float) -> tuple[float, str]:
         unit_step = 1024
         unit_step_thresh = unit_step - 0.5
 
-        unit = None
         for unit in unit_labels:
             if num < unit_step_thresh:
-                # Only return when under the rounding threshhold
+                # Only return when under the rounding threshold
                 break
             if unit != last_label:
                 num /= unit_step

tox.ini

@@ -1,13 +1,12 @@
 [tox]
-minversion = 3.8.0
-envlist = py38, py39, py310, flake8, mypy
+minversion = 3.10.0
+envlist = py310, py311, flake8, mypy
 isolated_build = true
 
 [gh-actions]
 python =
-    3.8: py38, mypy, flake8
-    3.9: py39
     3.10: py310
+    3.11: py311
 
 [testenv]
 setenv =
@@ -17,16 +16,16 @@ setenv =
 deps =
     -r{toxinidir}/requirements_dev.txt
 commands =
-    pytest --basetemp={envtmpdir}
+    pytest tests/api -vv -rA --basetemp={envtmpdir}
 
 [testenv:flake8]
-basepython = python3.8
+basepython = python3.10
 deps = flake8
-commands = flake8 src tests
+commands = flake8 src/isisdl/backend/crud.py src/isisdl/backend/models.py src/isisdl/api src/isisdl/db_conf.py src/isisdl/__main__.py src/isisdl/settings.py src/isisdl/utils.py tests/api
 
 [testenv:mypy]
-basepython = python3.8
+basepython = python3.10
 deps =
     -r{toxinidir}/requirements_dev.txt
-commands = mypy src
+commands = mypy src/isisdl/backend/crud.py src/isisdl/backend/models.py src/isisdl/api src/isisdl/db_conf.py src/isisdl/__main__.py src/isisdl/settings.py src/isisdl/utils.py tests/api