Rewrite pagination query to use subquery instead of join

server/mergin/sync/permissions.py

@@ -10,6 +10,7 @@
 from sqlalchemy import or_
 
 from .utils import is_valid_uuid
+from ..app import db
 from ..auth.models import User
 from .models import Project, Upload, ProjectRole, ProjectUser
 
@@ -62,10 +63,8 @@ def query(cls, user, as_admin=True, public=True):
             if user.is_authenticated and user.is_admin and as_admin:
                 return Project.query
 
-            query = (
-                Project.query.join(ProjectUser)
-                .filter(Project.storage_params.isnot(None))
-                .filter(Project.removed_at.is_(None))
+            query = Project.query.filter(Project.storage_params.isnot(None)).filter(
+                Project.removed_at.is_(None)
             )
             if user.is_authenticated and user.active:
                 all_workspaces = current_app.ws_handler.list_user_workspaces(
@@ -76,19 +75,24 @@ def query(cls, user, as_admin=True, public=True):
                     for ws in all_workspaces
                     if ws.user_has_permissions(user, "read")
                 ]
+                subquery = (
+                    db.session.query(ProjectUser.project_id)
+                    .filter(ProjectUser.user_id == user.id)
+                    .subquery()
+                )
                 if public:
                     query = query.filter(
                         or_(
                             Project.public.is_(True),
                             Project.workspace_id.in_(user_workspace_ids),
-                            ProjectUser.user_id == user.id,
+                            Project.id.in_(subquery),
                         )
                     )
                 else:
                     query = query.filter(
                         or_(
                             Project.workspace_id.in_(user_workspace_ids),
-                            ProjectUser.user_id == user.id,
+                            Project.id.in_(subquery),
                         )
                     )
             else:

server/mergin/sync/public_api_controller.py

@@ -643,14 +643,7 @@ def get_paginated_projects(
         only_public,
     )
     result = projects.paginate(page, per_page).items
-    # for count (done in subquery) we only need ids and then to do distinct for deduplications due to joins
-    # https://docs.sqlalchemy.org/en/20/faq/sessions.html#my-query-does-not-return-the-same-number-of-objects-as-query-count-tells-me-why
-    total = (
-        projects.order_by(None)
-        .options(load_only(Project.id))
-        .distinct(Project.id)
-        .count()
-    )
+    total = projects.paginate().total
 
     # create user map id:username passed to project schema to minimize queries to db
     projects_ids = [p.id for p in result]

server/mergin/sync/workspace.py

@@ -164,8 +164,7 @@ def filter_projects(
     ):
         if only_public:
             projects = (
-                Project.query.join(ProjectUser)
-                .filter(Project.storage_params.isnot(None))
+                Project.query.filter(Project.storage_params.isnot(None))
                 .filter(Project.removed_at.is_(None))
                 .filter(Project.public.is_(True))
             )
@@ -186,7 +185,12 @@ def filter_projects(
                     if workspace.user_has_permissions(user, "read"):
                         projects = projects.filter(Project.workspace_id == workspace.id)
                     else:
-                        projects = projects.filter(ProjectUser.user_id == user.id)
+                        subquery = (
+                            db.session.query(ProjectUser.project_id)
+                            .filter(ProjectUser.user_id == user.id)
+                            .subquery()
+                        )
+                        projects = projects.filter(Project.id.in_(subquery))
 
         if name:
             projects = projects.filter(Project.name.ilike("%{}%".format(name)))