Bump backend dependencies (#408)

Bump backend dependencies

.github/workflows/test_code.yml

@@ -24,12 +24,16 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: '3.10'
-      - name: install mypy==0.790
-        run: pip3 install mypy==0.790
+      - name: install mypy==1.2.0
+        run: pip3 install mypy==1.2.0
       - name: install requirements
         run: pip3 install -r requirements.txt
-      - name: run mypy
-        run: mypy src
+      - name: run mypy on main files
+        run: mypy src/app.py src/daemon.py
+      - name: run mypy on tests
+        run: MYPYPATH=src/ mypy src/tests/
+      - name: run mypy on utils
+        run: MYPYPATH=src/ mypy src/utils/
   test_python_style:
     name: python flake8
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -50,7 +50,7 @@ $ prettier --write src/mqueryfront/
 - Verify that there are no type errors with [mypy](http://mypy-lang.org/):
 
 ```bash
-$ pip install mypy==0.790
+$ pip install mypy==1.2.0
 $ mypy src
 ```
 

requirements.txt

@@ -1,28 +1,34 @@
-anyio==3.4.0
+alembic==1.11.1
+annotated-types==0.7.0
+anyio==4.6.0
 asgiref==3.4.1
-cachetools==4.2.4
-certifi==2021.10.8
-charset-normalizer==2.0.9
-click==8.0.3
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+click==8.1.7
+cryptography==43.0.1
 Deprecated==1.2.13
-fastapi==0.70.0
-h11==0.12.0
-idna==3.3
-pydantic==1.10.14
-pyzmq==24.0.1
-redis==4.0.2
-requests==2.26.0
-sniffio==1.2.0
-starlette==0.16.0
-typing-extensions==4.9.0
-urllib3==1.26.7
-uvicorn==0.15.0
-wrapt==1.13.3
-yara-python==4.1.3
-yaramod==3.12.1
-PyJWT[crypto]==2.3.0
-rq==1.11.1
-typed-config==1.3.2
-sqlmodel==0.0.11
+fastapi==0.115.0
+h11==0.14.0
+idna==3.10
 psycopg2==2.9.9
-alembic==1.11.1
+pycparser==2.22
+pydantic==1.10.18
+pydantic_core==2.23.4
+PyJWT[crypto]==2.9.0
+pyzmq==26.2.0
+redis==5.0.8
+requests==2.32.2
+rq==1.16.2
+sniffio==1.3.1
+sqlmodel==0.0.11
+starlette==0.38.6
+typed-config==2.0.3
+types-requests==2.32.0.20240914
+typing_extensions==4.12.2
+urllib3==2.2.3
+uvicorn==0.30.6
+wrapt==1.16.0
+yara-python==4.5.1
+yaramod==3.23.0

setup.cfg

@@ -3,7 +3,7 @@ ignore = E501,W503,E203
 exclude = mqueryfront/
 
 [mypy]
-python_version = 3.6
+python_version = 3.10
 
 [mypy-yaramod.*]
 ignore_missing_imports = True

src/app.py

@@ -122,7 +122,7 @@ async def current_user(authorization: Optional[str] = Header(None)) -> User:
 async def add_headers(request: Request, call_next: Callable) -> Response:
     response = await call_next(request)
     response.headers["X-Frame-Options"] = "deny"
-    response.headers["Access-Control-Allow-Origin"] = request.client.host
+    response.headers["Access-Control-Allow-Origin"] = request.client.host  # type: ignore
     response.headers[
         "Access-Control-Allow-Headers"
     ] = "cache-control,x-requested-with,content-type,authorization"

src/lib/ursadb.py

@@ -53,7 +53,7 @@ def __execute(self, command: str, recv_timeout: int = 2000) -> Json:
     def query(
         self,
         query: str,
-        taints: List[str] = None,
+        taints: List[str] | None = None,
         dataset: Optional[str] = None,
     ) -> Json:
         command = "select "

src/lib/yaraparse.py

@@ -1,7 +1,7 @@
 import argparse
 import itertools
 import re
-from typing import Any, Dict, List, Match, Optional
+from typing import Any, Dict, List, Match, Optional, cast, Callable
 
 from yaramod import (  # type: ignore
     AllExpression,
@@ -17,7 +17,6 @@
     OfExpression,
     OrExpression,
     ParenthesesExpression,
-    PlainString,
     Regexp,
     RegexpConcat,
     RegexpGroup,
@@ -358,7 +357,7 @@ def ursify_plain_string(
         return ursa_ascii
 
 
-def ursify_xor_string(string: PlainString) -> UrsaExpression:
+def ursify_xor_string(string: String) -> UrsaExpression:
     text_ascii = string.pure_text
     xored_strings: List[UrsaExpression] = []
 
@@ -389,14 +388,14 @@ def ursify_string(string: String) -> Optional[UrsaExpression]:
         value_safe = string.pure_text.decode()
         return ursify_hex(value_safe)
     elif string.is_regexp:
-        return ursify_regex_string(string)
+        return ursify_regex_string(cast(Regexp, string))
 
     return None
 
 
 class RuleParseEngine:
     def __init__(
-        self, strings: Dict[str, str], rules: Dict[str, YaraRuleData]
+        self, strings: Dict[str, String], rules: Dict[str, YaraRuleData]
     ) -> None:
         self.strings = strings
         self.rules = rules
@@ -545,7 +544,7 @@ def str_in_expr(
     ) -> Optional[UrsaExpression]:
         return ursify_string(self.strings[condition.id])
 
-    CONDITION_HANDLERS = {
+    CONDITION_HANDLERS: Dict[type, Callable] = {
         AndExpression: and_expr,
         OrExpression: or_expr,
         ParenthesesExpression: pare_expr,

src/models/agentgroup.py

@@ -3,23 +3,17 @@
 from ..models.jobagent import JobAgent
 
 
-class AgentGroupBase(SQLModel):
+class AgentGroupView(SQLModel):
     name: str
     ursadb_url: str
     plugins_spec: Dict[str, Dict[str, str]] = Field(sa_column=Column(JSON))
     active_plugins: List[str] = Field(sa_column=Column(ARRAY(String)))
 
 
-class AgentGroup(AgentGroupBase, table=True):
+class AgentGroup(AgentGroupView, table=True):
     """Agent group is a group of processes working on a single
     file group, with a shared storage, and a single backing ursadb.
     """
 
     id: Union[int, None] = Field(default=None, primary_key=True)
     jobs: List["JobAgent"] = Relationship(back_populates="agent")
-
-
-class AgentGroupView(AgentGroupBase):
-    """Pydantic model used in the public API."""
-
-    pass

src/models/job.py

@@ -6,8 +6,8 @@
     from ..models.jobagent import JobAgent
 
 
-class JobBase(SQLModel):
-    """Base class for entities related to mquery jobs."""
+class JobView(SQLModel):
+    """Public fields of mquery jobs."""
 
     id: str
     status: str
@@ -30,16 +30,10 @@ class JobBase(SQLModel):
     agents_left: int
 
 
-class Job(JobBase, table=True):
+class Job(JobView, table=True):
     """Job object in the database. Internal ID is an implementation detail."""
 
     internal_id: Union[int, None] = Field(default=None, primary_key=True)
 
     matches: List["Match"] = Relationship(back_populates="job")
     agents: List["JobAgent"] = Relationship(back_populates="job")
-
-
-class JobView(JobBase):
-    """Pydantic model used in the public API."""
-
-    pass

src/schema.py

@@ -1,12 +1,12 @@
 from enum import Enum
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Sequence
 from pydantic import BaseModel, Field  # type: ignore
 from .models.job import JobView
 from .models.agentgroup import AgentGroupView
 
 
 class JobsSchema(BaseModel):
-    jobs: List[JobView]
+    jobs: Sequence[JobView]
 
 
 class ConfigSchema(BaseModel):

src/tasks.py

@@ -165,7 +165,7 @@ def make_agent(group_override: Optional[str] = None):
     if group_override is not None:
         group_id = group_override
     else:
-        group_id = get_current_job().origin
+        group_id = get_current_job().origin  # type: ignore
     return Agent(group_id)
 
 

src/utils/index.py

@@ -287,7 +287,7 @@ def main() -> None:
             path_mount = args.path
 
         path = Path(args.path)
-        if not path.exists:
+        if not path.exists():
             logging.error("Path (--path) %s does not exist.", args.path)
             return