Implemented the RateLimiter. It limits the download rate by handing o…

…ut tokens to other Tasks.

Is tested with 95% Code coverage so it should be good

src/isisdl/__main__.py

@@ -2,16 +2,19 @@
 import asyncio
 import sys
 import time
+from asyncio import create_task
+from threading import Thread
 
 import isisdl.compress as compress
 from isisdl.api.crud import authenticate_new_session
 from isisdl.api.endpoints import CourseContentsAPI, UserCourseListAPI
+from isisdl.api.rate_limiter import RateLimiter, ThrottleType
 from isisdl.backend import sync_database
 from isisdl.backend.config import init_wizard, config_wizard
-from isisdl.backend.crud import read_config, read_user
+from isisdl.backend.crud import read_config, read_user, create_default_config, store_user
 from isisdl.backend.request_helper import CourseDownloader
 from isisdl.db_conf import init_database, DatabaseSessionMaker
-from isisdl.settings import is_first_time, is_static, forbidden_chars, has_ffmpeg, fstype, is_windows, working_dir_location, python_executable, is_macos, is_online
+from isisdl.settings import is_first_time, is_static, forbidden_chars, has_ffmpeg, fstype, is_windows, working_dir_location, python_executable, is_macos, is_online, DEBUG_ASSERTS
 from isisdl.utils import args, acquire_file_lock_or_exit, generate_error_message, install_latest_version, export_config, database_helper, config, migrate_database, Config, compare_download_diff
 from isisdl.version import __version__
 
@@ -32,6 +35,12 @@ def print_version() -> None:
 """)
 
 
+async def getter(id: int, limiter: RateLimiter) -> None:
+    while True:
+        token = await limiter.get(ThrottleType.free_for_all)
+        await asyncio.sleep(0.01)
+        print(f"Got token from task {id}!")
+
 async def _new_main() -> None:
     with DatabaseSessionMaker() as db:
         config = read_config(db)
@@ -51,8 +60,26 @@ async def _new_main() -> None:
         contents = await CourseContentsAPI.get(db, session, courses)
         print(f"{time.perf_counter() - s:.3f}s")
 
+        limiter = RateLimiter(20)
+        create_task(getter(1, limiter))
+        create_task(getter(2, limiter))
+        create_task(getter(3, limiter))
+
+
+        # TODO: How to deal with crashing threads
+        #   - Have a menu which enables 3 choices:
+        #     - restart with same file
+        #     - restart with next file
+        #     - ignore and keep the thread dead
+
+        await asyncio.sleep(50)
+
+        # TODO: Can I somehow move this to the __del__ method?
+        await session.session.close()
+
 
 def _main() -> None:
+
     init_database()
 
     if is_first_time:
@@ -110,6 +137,8 @@ def _main() -> None:
 
     asyncio.run(_new_main())
 
+    return
+
     install_latest_version()
 
     if args.update:

src/isisdl/api/crud.py

@@ -40,11 +40,12 @@ async def authenticate_new_session(user: User, config: Config) -> AuthenticatedS
     ) as response:
 
         # Check if authentication succeeded
-        if response is None or response.url == "https://shibboleth.tubit.tu-berlin.de/idp/profile/SAML2/Redirect/SSO?execution=e1s3":
+        if response is None or str(response.url) == "https://shibboleth.tubit.tu-berlin.de/idp/profile/SAML2/Redirect/SSO?execution=e1s3":
             return None
 
         # Extract the session key
-        _session_key = re.search(r"\"sesskey\":\"(.*?)\"", await response.text())
+        text =  await response.text()
+        _session_key = re.search(r"\"sesskey\":\"(.*?)\"", text)
         if _session_key is None:
             return None
 

src/isisdl/api/endpoints.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
 import asyncio
-import json
 from collections import defaultdict
 from json import JSONDecodeError
-from typing import Any, Self
+from typing import Any, Self, cast
 
 from sqlalchemy.orm import Session as DatabaseSession
 
@@ -59,6 +58,15 @@ async def _get(cls, session: AuthenticatedSession, data: dict[str, Any] | None =
 class UserIDAPI(APIEndpoint):
     function = "core_webservice_get_site_info"
 
+    @classmethod
+    async def get(cls, session: AuthenticatedSession) -> int | None:
+        response = await cls._get(session)
+
+        if response is None:
+            return None
+
+        return cast(int, response["userid"])
+
 
 class UserCourseListAPI(APIEndpoint):
     function = "core_enrol_get_users_courses"
@@ -206,10 +214,10 @@ async def get(cls, db: DatabaseSession, session: AuthenticatedSession, courses:
 
         files = cls._filter_duplicates_from_files(normalized_files_with_duplicates)
 
-        existing_containers = {(it.course_id, it.url): it for it in read_downloadable_media_containers(db)}
+        existing_containers = {(it.course_id, normalize_url(it.url)): it for it in read_downloadable_media_containers(db)}
 
         return add_or_update_objects_to_database(
-            db, existing_containers, files, DownloadableMediaContainer, lambda x: (x["course_id"], x["fileurl"]),
+            db, existing_containers, files, DownloadableMediaContainer, lambda x: (x["course_id"], normalize_url(x["fileurl"])),
             {"url": "fileurl", "course_id": "course_id", "media_type": "media_type", "relative_path": "relative_path",
              "name": "filename", "size": "filesize", "time_created": "timecreated", "time_modified": "timemodified"},
             {"url": normalize_url, "time_created": datetime_fromtimestamp_with_None, "time_modified": datetime_fromtimestamp_with_None}
@@ -228,19 +236,15 @@ async def old_get(cls, db: DatabaseSession, session: AuthenticatedSession, cours
             if response is None:
                 continue
 
-            course_id: int = response["course_id"]
-            course_contents: list[dict[str, Any]] = response["it"]
+            course_contents, course_id = response
 
             # Unfortunately, it doesn't seam as if python supports matching of nested dicts / lists
             for week in course_contents:
                 match week:
                     case {"modules": modules}:
                         for module in modules:
                             match module:
-                                case {"url": url, "contents": files}:
-                                    if isis_ignore.match(url) is not None:
-                                        continue
-
+                                case {"contents": files}:
                                     for file in files:
                                         match file:
                                             case {"fileurl": url, "type": file_type, "filepath": relative_path}:

src/isisdl/api/models.py

@@ -149,3 +149,6 @@ def head(self, url: str, **kwargs: Any) -> _RequestContextManager | Error:
                 time.sleep(download_static_sleep_time)
 
         return Error()
+
+    async def close(self) -> None:
+        await self.session.close()