Resolve conflicts

.prod.env

@@ -177,3 +177,5 @@ GLOBAL_STORAGE=10737418240
 
 # Gunicorn server socket
 PORT=5000
+
+GEVENT_WORKER=True

server/application.py

@@ -9,14 +9,17 @@
 # Modules that had direct imports (NOT patched): ['urllib3.util, 'urllib3.util.ssl']"
 # which comes from using requests (its deps) lib in webhooks
 
-import os
-from random import randint
+from mergin.config import Configuration as MainConfig
 
-if not os.getenv("NO_MONKEY_PATCH", False):
+if MainConfig.GEVENT_WORKER:
     import gevent.monkey
+    import psycogreen.gevent
+
+    gevent.monkey.patch_all()
+    psycogreen.gevent.patch_psycopg()
 
-    gevent.monkey.patch_all(subprocess=True)
 
+from random import randint
 from celery.schedules import crontab
 from mergin.app import create_app
 from mergin.auth.tasks import anonymize_removed_users
@@ -28,6 +31,7 @@
 
 Configuration.SERVER_TYPE = "ce"
 Configuration.USER_SELF_REGISTRATION = False
+
 application = create_app(
     [
         "DOCS_URL",
@@ -37,6 +41,7 @@
         "GLOBAL_ADMIN",
         "GLOBAL_READ",
         "GLOBAL_WRITE",
+        "ENABLE_SUPERADMIN_ASSIGNMENT",
     ]
 )
 register_stats(application)

server/config.py

@@ -25,18 +25,6 @@
 """
 import logging
 
-try:
-    from psycogreen.gevent import patch_psycopg
-except ImportError:
-    import sys
-    import traceback
-
-    exception_info = traceback.format_exc()
-    sys.stderr.write(
-        f"Failed to load required functions from the psycogreen library: { exception_info }\n"
-    )
-    sys.exit(1)
-
 worker_class = "gevent"
 
 workers = 2
@@ -59,12 +47,7 @@
 
 max_requests_jitter = 5000
 
-
-def do_post_fork(server, worker):
-    patch_psycopg()
-
-
-post_fork = do_post_fork
+timeout = 30
 
 
 """

server/mergin/__init__.py

@@ -1,5 +1,3 @@
 # Copyright (C) Lutra Consulting Limited
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
-
-from .app import db, mail, ma, create_app

server/mergin/app.py

@@ -7,11 +7,12 @@
 import os
 import connexion
 import wtforms_json
+import gevent
 from marshmallow import fields
 from sqlalchemy.schema import MetaData
 from flask_sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
-from flask import json, jsonify, request, abort, current_app, Flask
+from flask import json, jsonify, request, abort, current_app, Flask, Request, Response
 from flask_login import current_user, LoginManager
 from flask_wtf.csrf import generate_csrf, CSRFProtect
 from flask_migrate import Migrate
@@ -27,6 +28,7 @@
 from typing import List, Dict, Optional
 
 from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
+from .config import Configuration
 
 convention = {
     "ix": "ix_%(column_0_label)s",
@@ -105,9 +107,24 @@ def update_obj(self, obj):
                 field.populate_obj(obj, name)
 
 
-def create_simple_app() -> Flask:
-    from .config import Configuration
+class GeventTimeoutMiddleware:
+    """Middleware to implement gevent.Timeout() for all requests"""
+
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        request = Request(environ)
+        try:
+            with gevent.Timeout(Configuration.GEVENT_REQUEST_TIMEOUT):
+                return self.app(environ, start_response)
+        except gevent.Timeout:
+            logging.error(f"Gevent worker: Request {request.path} timed out")
+            resp = Response("Gateway Timeout", mimetype="text/plain", status=504)
+            return resp(environ, start_response)
+
 
+def create_simple_app() -> Flask:
     app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir))
     flask_app = app.app
 
@@ -117,6 +134,9 @@ def create_simple_app() -> Flask:
     ma.init_app(flask_app)
     Migrate(flask_app, db)
     flask_app.connexion_app = app
+    # in case of gevent worker type use middleware to implement custom request timeout
+    if Configuration.GEVENT_WORKER:
+        flask_app.wsgi_app = GeventTimeoutMiddleware(flask_app.wsgi_app)
 
     @flask_app.cli.command()
     def init_db():
@@ -133,7 +153,6 @@ def init_db():
 def create_app(public_keys: List[str] = None) -> Flask:
     """Factory function to create Flask app instance"""
     from itsdangerous import BadTimeSignature, BadSignature
-    from .config import Configuration
     from .auth import auth_required, decode_token
     from .auth.models import User
 

server/mergin/auth/api.yaml

@@ -171,7 +171,7 @@ paths:
                   count:
                     type: integer
                     example: 10
-                  users:
+                  items:
                     type: array
                     items:
                       $ref: "#/components/schemas/User"
@@ -670,6 +670,11 @@ components:
           type: string
           format: date-time
           example: 2023-07-30T08:47:58Z
+        registration_date:
+          nullable: true
+          type: string
+          format: date-time
+          example: 2023-07-30T08:47:58Z
         profile:
           $ref: "#/components/schemas/UserProfile"
     PaginatedUsers:
@@ -839,4 +844,4 @@ components:
         - editor
         - reader
         - guest
-      example: reader
+      example: reader

server/mergin/auth/app.py

@@ -12,7 +12,7 @@
 from .commands import add_commands
 from .config import Configuration
 from .models import User, UserProfile
-from .. import db
+from ..app import db
 
 # signal for other versions to listen to
 user_account_closed = signal("user_account_closed")

server/mergin/auth/commands.py

@@ -6,7 +6,7 @@
 from flask import Flask
 from sqlalchemy import or_, func
 
-from .. import db
+from ..app import db
 from .models import User, UserProfile
 
 

server/mergin/auth/controller.py

@@ -6,7 +6,7 @@
 import pytz
 from datetime import datetime, timedelta
 from connexion import NoContent
-from sqlalchemy import func, desc, asc
+from sqlalchemy import func, desc, asc, or_
 from sqlalchemy.sql.operators import is_
 from flask import request, current_app, jsonify, abort, render_template
 from flask_login import login_user, logout_user, current_user
@@ -23,7 +23,7 @@
     user_account_closed,
 )
 from .bearer import encode_token
-from .models import User, LoginHistory
+from .models import User, LoginHistory, UserProfile
 from .schemas import UserSchema, UserSearchSchema, UserProfileSchema, UserInfoSchema
 from .forms import (
     LoginForm,
@@ -35,8 +35,7 @@
     UserChangePasswordForm,
     ApiLoginForm,
 )
-from .. import db
-from ..app import DEPRECATION_API_MSG
+from ..app import DEPRECATION_API_MSG, db
 from ..utils import format_time_delta
 
 
@@ -408,11 +407,17 @@ def update_user(username):  # pylint: disable=W0613,W0612
     form = UserForm.from_json(request.json)
     if not form.validate_on_submit():
         return jsonify(form.errors), 400
+    if request.json.get("is_admin") is not None and not current_app.config.get(
+        "ENABLE_SUPERADMIN_ASSIGNMENT"
+    ):
+        abort(400, "Unable to assign super admin role")
 
     user = User.query.filter_by(username=username).first_or_404("User not found")
     form.update_obj(user)
+
     # remove inactive since flag for ban or re-activation
     user.inactive_since = None
+
     db.session.add(user)
     db.session.commit()
     return jsonify(UserSchema().dump(user))
@@ -449,13 +454,17 @@ def get_paginated_users(
 
     :rtype: Dict[str: List[User], str: Integer]
     """
-    users = User.query.filter(
+    users = User.query.join(UserProfile).filter(
         is_(User.username.ilike("deleted_%"), False) | is_(User.active, True)
     )
 
     if like:
-        attr = User.email if "@" in like else User.username
-        users = users.filter(attr.ilike(f"%{like}%"))
+        users = users.filter(
+            User.username.ilike(f"%{like}%")
+            | User.email.ilike(f"%{like}%")
+            | UserProfile.first_name.ilike(f"%{like}%")
+            | UserProfile.last_name.ilike(f"%{like}%")
+        )
 
     if descending and order_by:
         users = users.order_by(desc(User.__table__.c[order_by]))
@@ -467,7 +476,7 @@ def get_paginated_users(
 
     result_users = UserSchema(many=True).dump(result)
 
-    data = {"users": result_users, "total": total}
+    data = {"items": result_users, "count": total}
     return data, 200
 
 

server/mergin/auth/models.py

@@ -9,7 +9,7 @@
 from flask import current_app, request
 from sqlalchemy import or_, func
 
-from .. import db
+from ..app import db
 from ..sync.utils import get_user_agent, get_ip, get_device_id
 
 

server/mergin/auth/schemas.py

@@ -5,9 +5,8 @@
 from flask import current_app
 from marshmallow import fields
 
-from .. import ma
 from .models import User, UserProfile
-from ..app import DateTimeWithZ
+from ..app import DateTimeWithZ, ma
 
 
 class UserProfileSchema(ma.SQLAlchemyAutoSchema):
@@ -71,6 +70,7 @@ class Meta:
             "verified_email",
             "profile",
             "scheduled_removal",
+            "registration_date",
         )
         load_instance = True
 

server/mergin/auth/tasks.py

@@ -6,7 +6,7 @@
 from sqlalchemy.sql.operators import isnot
 
 from ..celery import celery
-from .. import db
+from .app import db
 from .models import User
 from .config import Configuration
 

server/mergin/celery.py

@@ -9,7 +9,7 @@
 from smtplib import SMTPException, SMTPServerDisconnected
 
 from .config import Configuration
-from . import mail
+from .app import mail
 
 
 # create on flask app independent object

server/mergin/config.py

@@ -3,10 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
 import os
-from .version import get_version
 from decouple import config, Csv
 
-config_dir = os.path.abspath(os.path.dirname(__file__))
+from .version import get_version
 
 
 class Configuration(object):
@@ -95,6 +94,16 @@ class Configuration(object):
 
     # build hash number
     BUILD_HASH = config("BUILD_HASH", default="")
+
+    # Allow changing access to admin panel
+    ENABLE_SUPERADMIN_ASSIGNMENT = config(
+        "ENABLE_SUPERADMIN_ASSIGNMENT", default=True, cast=bool
+    )
     # backend version
     VERSION = config("VERSION", default=get_version())
     SERVER_TYPE = config("SERVER_TYPE", default="ce")
+
+    # whether to run flask app with gevent worker type in gunicorn
+    # using gevent type of worker impose some requirements on code, e.g. to be greenlet safe, custom timeouts
+    GEVENT_WORKER = config("GEVENT_WORKER", default=False, cast=bool)
+    GEVENT_REQUEST_TIMEOUT = config("GEVENT_REQUEST_TIMEOUT", default=30, cast=int)

server/mergin/stats/models.py

@@ -5,7 +5,7 @@
 import uuid
 from sqlalchemy.dialects.postgresql import UUID
 
-from .. import db
+from ..app import db
 
 
 class MerginInfo(db.Model):

server/mergin/sync/commands.py

@@ -10,7 +10,7 @@
 from flask import Flask, current_app
 
 from .files import UploadChanges
-from .. import db
+from ..app import db
 from .models import Project, ProjectAccess, ProjectVersion
 from .utils import split_project_path
 from ..auth.models import User
@@ -62,7 +62,7 @@ def create(name, namespace, username):  # pylint: disable=W0612
 
     @project.command()
     @click.argument("project-name")
-    @click.option("--version", required=True)
+    @click.option("--version", type=int, required=True)
     @click.option("--directory", type=click.Path(), required=True)
     def download(project_name, version, directory):  # pylint: disable=W0612
         """Download files for project at particular version"""

server/mergin/sync/db_events.py

@@ -6,7 +6,7 @@
 from flask import current_app, abort
 from sqlalchemy import event
 
-from .. import db
+from ..app import db
 
 
 def check(session):

server/mergin/sync/files.py

@@ -8,8 +8,7 @@
 from marshmallow import fields, EXCLUDE, pre_load, post_load, post_dump
 from pathvalidate import sanitize_filename
 
-from .. import ma
-from ..app import DateTimeWithZ
+from ..app import DateTimeWithZ, ma
 
 
 def mergin_secure_filename(filename: str) -> str: