diff --git a/.prod.env b/.prod.env index a9ef451f..da2011a0 100644 --- a/.prod.env +++ b/.prod.env @@ -177,3 +177,5 @@ GLOBAL_STORAGE=10737418240 # Gunicorn server socket PORT=5000 + +GEVENT_WORKER=True diff --git a/server/application.py b/server/application.py index d1eae870..2a4220b6 100644 --- a/server/application.py +++ b/server/application.py @@ -9,14 +9,17 @@ # Modules that had direct imports (NOT patched): ['urllib3.util, 'urllib3.util.ssl']" # which comes from using requests (its deps) lib in webhooks -import os -from random import randint +from mergin.config import Configuration as MainConfig -if not os.getenv("NO_MONKEY_PATCH", False): +if MainConfig.GEVENT_WORKER: import gevent.monkey + import psycogreen.gevent + + gevent.monkey.patch_all() + psycogreen.gevent.patch_psycopg() - gevent.monkey.patch_all(subprocess=True) +from random import randint from celery.schedules import crontab from mergin.app import create_app from mergin.auth.tasks import anonymize_removed_users @@ -28,6 +31,7 @@ Configuration.SERVER_TYPE = "ce" Configuration.USER_SELF_REGISTRATION = False + application = create_app( [ "DOCS_URL", @@ -37,6 +41,7 @@ "GLOBAL_ADMIN", "GLOBAL_READ", "GLOBAL_WRITE", + "ENABLE_SUPERADMIN_ASSIGNMENT", ] ) register_stats(application) diff --git a/server/config.py b/server/config.py index 7c291f20..7c122dc3 100644 --- a/server/config.py +++ b/server/config.py @@ -25,18 +25,6 @@ """ import logging -try: - from psycogreen.gevent import patch_psycopg -except ImportError: - import sys - import traceback - - exception_info = traceback.format_exc() - sys.stderr.write( - f"Failed to load required functions from the psycogreen library: { exception_info }\n" - ) - sys.exit(1) - worker_class = "gevent" workers = 2 @@ -59,12 +47,7 @@ max_requests_jitter = 5000 - -def do_post_fork(server, worker): - patch_psycopg() - - -post_fork = do_post_fork +timeout = 30 """ diff --git a/server/mergin/__init__.py b/server/mergin/__init__.py index b1d7cb3f..f8ea3438 100644 --- a/server/mergin/__init__.py +++ b/server/mergin/__init__.py @@ -1,5 +1,3 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial - -from .app import db, mail, ma, create_app diff --git a/server/mergin/app.py b/server/mergin/app.py index 4460e010..751b6561 100644 --- a/server/mergin/app.py +++ b/server/mergin/app.py @@ -7,11 +7,12 @@ import os import connexion import wtforms_json +import gevent from marshmallow import fields from sqlalchemy.schema import MetaData from flask_sqlalchemy import SQLAlchemy from flask_marshmallow import Marshmallow -from flask import json, jsonify, request, abort, current_app, Flask +from flask import json, jsonify, request, abort, current_app, Flask, Request, Response from flask_login import current_user, LoginManager from flask_wtf.csrf import generate_csrf, CSRFProtect from flask_migrate import Migrate @@ -27,6 +28,7 @@ from typing import List, Dict, Optional from .sync.utils import get_blacklisted_dirs, get_blacklisted_files +from .config import Configuration convention = { "ix": "ix_%(column_0_label)s", @@ -105,9 +107,24 @@ def update_obj(self, obj): field.populate_obj(obj, name) -def create_simple_app() -> Flask: - from .config import Configuration +class GeventTimeoutMiddleware: + """Middleware to implement gevent.Timeout() for all requests""" + + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + request = Request(environ) + try: + with gevent.Timeout(Configuration.GEVENT_REQUEST_TIMEOUT): + return self.app(environ, start_response) + except gevent.Timeout: + logging.error(f"Gevent worker: Request {request.path} timed out") + resp = Response("Gateway Timeout", mimetype="text/plain", status=504) + return resp(environ, start_response) + +def create_simple_app() -> Flask: app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir)) flask_app = app.app @@ -117,6 +134,9 @@ def create_simple_app() -> Flask: ma.init_app(flask_app) Migrate(flask_app, db) flask_app.connexion_app = app + # in case of gevent worker type use middleware to implement custom request timeout + if Configuration.GEVENT_WORKER: + flask_app.wsgi_app = GeventTimeoutMiddleware(flask_app.wsgi_app) @flask_app.cli.command() def init_db(): @@ -133,7 +153,6 @@ def init_db(): def create_app(public_keys: List[str] = None) -> Flask: """Factory function to create Flask app instance""" from itsdangerous import BadTimeSignature, BadSignature - from .config import Configuration from .auth import auth_required, decode_token from .auth.models import User diff --git a/server/mergin/auth/api.yaml b/server/mergin/auth/api.yaml index 337dc862..f89e110d 100644 --- a/server/mergin/auth/api.yaml +++ b/server/mergin/auth/api.yaml @@ -171,7 +171,7 @@ paths: count: type: integer example: 10 - users: + items: type: array items: $ref: "#/components/schemas/User" @@ -670,6 +670,11 @@ components: type: string format: date-time example: 2023-07-30T08:47:58Z + registration_date: + nullable: true + type: string + format: date-time + example: 2023-07-30T08:47:58Z profile: $ref: "#/components/schemas/UserProfile" PaginatedUsers: @@ -839,4 +844,4 @@ components: - editor - reader - guest - example: reader \ No newline at end of file + example: reader diff --git a/server/mergin/auth/app.py b/server/mergin/auth/app.py index 78363361..1a3caba4 100644 --- a/server/mergin/auth/app.py +++ b/server/mergin/auth/app.py @@ -12,7 +12,7 @@ from .commands import add_commands from .config import Configuration from .models import User, UserProfile -from .. import db +from ..app import db # signal for other versions to listen to user_account_closed = signal("user_account_closed") diff --git a/server/mergin/auth/commands.py b/server/mergin/auth/commands.py index 1f690cf7..80cc7fb8 100644 --- a/server/mergin/auth/commands.py +++ b/server/mergin/auth/commands.py @@ -6,7 +6,7 @@ from flask import Flask from sqlalchemy import or_, func -from .. import db +from ..app import db from .models import User, UserProfile diff --git a/server/mergin/auth/controller.py b/server/mergin/auth/controller.py index 36932def..cdea3181 100644 --- a/server/mergin/auth/controller.py +++ b/server/mergin/auth/controller.py @@ -6,7 +6,7 @@ import pytz from datetime import datetime, timedelta from connexion import NoContent -from sqlalchemy import func, desc, asc +from sqlalchemy import func, desc, asc, or_ from sqlalchemy.sql.operators import is_ from flask import request, current_app, jsonify, abort, render_template from flask_login import login_user, logout_user, current_user @@ -23,7 +23,7 @@ user_account_closed, ) from .bearer import encode_token -from .models import User, LoginHistory +from .models import User, LoginHistory, UserProfile from .schemas import UserSchema, UserSearchSchema, UserProfileSchema, UserInfoSchema from .forms import ( LoginForm, @@ -35,8 +35,7 @@ UserChangePasswordForm, ApiLoginForm, ) -from .. import db -from ..app import DEPRECATION_API_MSG +from ..app import DEPRECATION_API_MSG, db from ..utils import format_time_delta @@ -408,11 +407,17 @@ def update_user(username): # pylint: disable=W0613,W0612 form = UserForm.from_json(request.json) if not form.validate_on_submit(): return jsonify(form.errors), 400 + if request.json.get("is_admin") is not None and not current_app.config.get( + "ENABLE_SUPERADMIN_ASSIGNMENT" + ): + abort(400, "Unable to assign super admin role") user = User.query.filter_by(username=username).first_or_404("User not found") form.update_obj(user) + # remove inactive since flag for ban or re-activation user.inactive_since = None + db.session.add(user) db.session.commit() return jsonify(UserSchema().dump(user)) @@ -449,13 +454,17 @@ def get_paginated_users( :rtype: Dict[str: List[User], str: Integer] """ - users = User.query.filter( + users = User.query.join(UserProfile).filter( is_(User.username.ilike("deleted_%"), False) | is_(User.active, True) ) if like: - attr = User.email if "@" in like else User.username - users = users.filter(attr.ilike(f"%{like}%")) + users = users.filter( + User.username.ilike(f"%{like}%") + | User.email.ilike(f"%{like}%") + | UserProfile.first_name.ilike(f"%{like}%") + | UserProfile.last_name.ilike(f"%{like}%") + ) if descending and order_by: users = users.order_by(desc(User.__table__.c[order_by])) @@ -467,7 +476,7 @@ def get_paginated_users( result_users = UserSchema(many=True).dump(result) - data = {"users": result_users, "total": total} + data = {"items": result_users, "count": total} return data, 200 diff --git a/server/mergin/auth/models.py b/server/mergin/auth/models.py index 0c5ab341..58c706ad 100644 --- a/server/mergin/auth/models.py +++ b/server/mergin/auth/models.py @@ -9,7 +9,7 @@ from flask import current_app, request from sqlalchemy import or_, func -from .. import db +from ..app import db from ..sync.utils import get_user_agent, get_ip, get_device_id diff --git a/server/mergin/auth/schemas.py b/server/mergin/auth/schemas.py index bcebca5c..ea0bcd71 100644 --- a/server/mergin/auth/schemas.py +++ b/server/mergin/auth/schemas.py @@ -5,9 +5,8 @@ from flask import current_app from marshmallow import fields -from .. import ma from .models import User, UserProfile -from ..app import DateTimeWithZ +from ..app import DateTimeWithZ, ma class UserProfileSchema(ma.SQLAlchemyAutoSchema): @@ -71,6 +70,7 @@ class Meta: "verified_email", "profile", "scheduled_removal", + "registration_date", ) load_instance = True diff --git a/server/mergin/auth/tasks.py b/server/mergin/auth/tasks.py index cd91e153..a09848a8 100644 --- a/server/mergin/auth/tasks.py +++ b/server/mergin/auth/tasks.py @@ -6,7 +6,7 @@ from sqlalchemy.sql.operators import isnot from ..celery import celery -from .. import db +from .app import db from .models import User from .config import Configuration diff --git a/server/mergin/celery.py b/server/mergin/celery.py index 156e14bf..8442924d 100644 --- a/server/mergin/celery.py +++ b/server/mergin/celery.py @@ -9,7 +9,7 @@ from smtplib import SMTPException, SMTPServerDisconnected from .config import Configuration -from . import mail +from .app import mail # create on flask app independent object diff --git a/server/mergin/config.py b/server/mergin/config.py index 177063b5..94130176 100644 --- a/server/mergin/config.py +++ b/server/mergin/config.py @@ -3,10 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial import os -from .version import get_version from decouple import config, Csv -config_dir = os.path.abspath(os.path.dirname(__file__)) +from .version import get_version class Configuration(object): @@ -95,6 +94,16 @@ class Configuration(object): # build hash number BUILD_HASH = config("BUILD_HASH", default="") + + # Allow changing access to admin panel + ENABLE_SUPERADMIN_ASSIGNMENT = config( + "ENABLE_SUPERADMIN_ASSIGNMENT", default=True, cast=bool + ) # backend version VERSION = config("VERSION", default=get_version()) SERVER_TYPE = config("SERVER_TYPE", default="ce") + + # whether to run flask app with gevent worker type in gunicorn + # using gevent type of worker impose some requirements on code, e.g. to be greenlet safe, custom timeouts + GEVENT_WORKER = config("GEVENT_WORKER", default=False, cast=bool) + GEVENT_REQUEST_TIMEOUT = config("GEVENT_REQUEST_TIMEOUT", default=30, cast=int) diff --git a/server/mergin/stats/models.py b/server/mergin/stats/models.py index 6d6ffbbd..320e0137 100644 --- a/server/mergin/stats/models.py +++ b/server/mergin/stats/models.py @@ -5,7 +5,7 @@ import uuid from sqlalchemy.dialects.postgresql import UUID -from .. import db +from ..app import db class MerginInfo(db.Model): diff --git a/server/mergin/sync/commands.py b/server/mergin/sync/commands.py index dab796cc..a71be8fe 100644 --- a/server/mergin/sync/commands.py +++ b/server/mergin/sync/commands.py @@ -10,7 +10,7 @@ from flask import Flask, current_app from .files import UploadChanges -from .. import db +from ..app import db from .models import Project, ProjectAccess, ProjectVersion from .utils import split_project_path from ..auth.models import User @@ -62,7 +62,7 @@ def create(name, namespace, username): # pylint: disable=W0612 @project.command() @click.argument("project-name") - @click.option("--version", required=True) + @click.option("--version", type=int, required=True) @click.option("--directory", type=click.Path(), required=True) def download(project_name, version, directory): # pylint: disable=W0612 """Download files for project at particular version""" diff --git a/server/mergin/sync/db_events.py b/server/mergin/sync/db_events.py index dbba5ed3..18d1ce60 100644 --- a/server/mergin/sync/db_events.py +++ b/server/mergin/sync/db_events.py @@ -6,7 +6,7 @@ from flask import current_app, abort from sqlalchemy import event -from .. import db +from ..app import db def check(session): diff --git a/server/mergin/sync/files.py b/server/mergin/sync/files.py index 204edf86..12b30afe 100644 --- a/server/mergin/sync/files.py +++ b/server/mergin/sync/files.py @@ -8,8 +8,7 @@ from marshmallow import fields, EXCLUDE, pre_load, post_load, post_dump from pathvalidate import sanitize_filename -from .. import ma -from ..app import DateTimeWithZ +from ..app import DateTimeWithZ, ma def mergin_secure_filename(filename: str) -> str: diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 30c244a0..688021fd 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -24,13 +24,12 @@ from .files import ( File, - UploadFile, UploadChanges, ChangesSchema, ProjectFile, ) from .storages.disk import move_to_tmp -from .. import db +from ..app import db from .storages import DiskStorage from .utils import is_versioned_file, is_qgis diff --git a/server/mergin/sync/private_api.yaml b/server/mergin/sync/private_api.yaml index 4ab4b9b2..4160ed07 100644 --- a/server/mergin/sync/private_api.yaml +++ b/server/mergin/sync/private_api.yaml @@ -183,20 +183,13 @@ paths: - $ref: "#/components/parameters/Page" - $ref: "#/components/parameters/PerPage" - $ref: "#/components/parameters/OrderParams" - - name: name + - name: like in: query - description: Filter projects by name with ilike pattern + description: Filter projects by name or workspace name with ilike pattern required: false schema: type: string example: survey - - name: workspace - in: query - description: Filter projects by workspace with ilike pattern - required: false - schema: - type: string - example: my-workspace responses: "200": description: List of projects @@ -209,7 +202,7 @@ paths: type: integer description: Total number of all projects example: 20 - projects: + items: type: array items: $ref: "#/components/schemas/ProjectListItem" @@ -292,7 +285,7 @@ paths: id: type: string format: uuid - example: 'd4ecda97-0595-40af-892c-e7522de70bd2' + example: "d4ecda97-0595-40af-892c-e7522de70bd2" name: type: string example: survey @@ -460,7 +453,7 @@ components: - detail UsersLimitHit: allOf: - - $ref: '#/components/schemas/CustomError' + - $ref: "#/components/schemas/CustomError" type: object properties: rejected_emails: @@ -473,7 +466,7 @@ components: example: code: UsersLimitHit detail: Maximum number of people in this workspace is reached. Please upgrade your subscription to add more people (UsersLimitHit) - rejected_emails: [ rejected@example.com ] + rejected_emails: [rejected@example.com] users_quota: 6 ProjectAccessRequestList: type: array @@ -488,7 +481,7 @@ components: project_id: type: string format: uuid - example: 'd4ecda97-0595-40af-892c-e7522de70bd2' + example: "d4ecda97-0595-40af-892c-e7522de70bd2" project_name: type: string example: survey @@ -619,25 +612,25 @@ components: nullable: false items: type: string - example: [ john.doe ] + example: [john.doe] writersnames: type: array nullable: false items: type: string - example: [ john.doe ] + example: [john.doe] editorsnames: type: array nullable: false items: type: string - example: [ john.doe ] + example: [john.doe] readersnames: nullable: false type: array items: type: string - example: [ john.doe ] + example: [john.doe] public: type: boolean example: true diff --git a/server/mergin/sync/private_api_controller.py b/server/mergin/sync/private_api_controller.py index c6122166..01c5af6e 100644 --- a/server/mergin/sync/private_api_controller.py +++ b/server/mergin/sync/private_api_controller.py @@ -8,7 +8,7 @@ from sqlalchemy.orm import defer from sqlalchemy import text, and_, desc, asc -from .. import db +from ..app import db from ..auth import auth_required from ..auth.models import User, UserProfile from .forms import AccessPermissionForm @@ -185,10 +185,8 @@ def list_namespace_project_access_requests( @auth_required(permissions=["admin"]) -def list_projects( - page, per_page, order_params=None, name=None, workspace=None -): # noqa: E501 - projects = current_app.ws_handler.projects_query(name, workspace) +def list_projects(page, per_page, order_params=None, like=None): # noqa: E501 + projects = current_app.ws_handler.projects_query(like) # do not fetch from db what is not needed projects = projects.options( defer(Project.storage_params), @@ -211,7 +209,7 @@ def list_projects( result = projects.paginate(page, per_page).items total = projects.paginate(page, per_page).total data = AdminProjectSchema(many=True).dump(result) - data = {"projects": data, "count": total} + data = {"items": data, "count": total} return data, 200 diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 7a3b1833..b9e07552 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -32,7 +32,7 @@ from gevent import sleep import base64 from werkzeug.exceptions import HTTPException -from .. import db +from ..app import db from ..auth import auth_required from ..auth.models import User from .models import ( @@ -86,7 +86,8 @@ from .errors import StorageLimitHit from ..utils import format_time_delta -push_triggered = signal("push_triggered") +push_finished = signal("push_finished") +# TODO: Move to database events to handle all commits to project versions project_version_created = signal("project_version_created") @@ -195,6 +196,7 @@ def add_project(namespace): # noqa: E501 p = Project(**request.json, creator=current_user, workspace=workspace) p.updated = datetime.utcnow() + db.session.add(p) pa = ProjectAccess(p, public=request.json.get("public", False)) template_name = request.json.get("template", None) @@ -230,7 +232,6 @@ def add_project(namespace): # noqa: E501 get_device_id(request), ) - db.session.add(p) db.session.add(pa) db.session.add(version) db.session.commit() @@ -732,7 +733,6 @@ def project_push(namespace, project_name): if not ws: abort(404) - push_triggered.send(project) # fixme use get_latest pv = ProjectVersion.query.filter_by( project_id=project.id, name=project.latest_version @@ -874,6 +874,7 @@ def project_push(namespace, project_name): f"Transaction id: {upload.id}. No upload." ) project_version_created.send(pv) + push_finished.send(pv) return jsonify(ProjectSchema().dump(project)), 200 except IntegrityError as err: db.session.rollback() @@ -1084,6 +1085,7 @@ def push_finish(transaction_id): f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}." ) project_version_created.send(pv) + push_finished.send(pv) except (psycopg2.Error, FileNotFoundError, DataSyncError, IntegrityError) as err: db.session.rollback() logging.exception( @@ -1186,6 +1188,7 @@ def clone_project(namespace, project_name): # noqa: E501 workspace=ws, ) p.updated = datetime.utcnow() + db.session.add(p) pa = ProjectAccess(p, public=False) try: @@ -1211,7 +1214,6 @@ def clone_project(namespace, project_name): # noqa: E501 user_agent, device_id, ) - db.session.add(p) db.session.add(pa) db.session.add(project_version) db.session.commit() diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 8e567238..59cdbb2d 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -8,7 +8,7 @@ from flask import abort, jsonify from flask_login import current_user -from mergin import db +from mergin.app import db from mergin.auth import auth_required from mergin.sync.models import Project from mergin.sync.permissions import ProjectPermissions, require_project_by_uuid diff --git a/server/mergin/sync/schemas.py b/server/mergin/sync/schemas.py index 827391a9..d8c766aa 100644 --- a/server/mergin/sync/schemas.py +++ b/server/mergin/sync/schemas.py @@ -7,11 +7,10 @@ from flask_login import current_user from flask import current_app -from .. import ma from .files import ProjectFileSchema, FileSchema from .permissions import ProjectPermissions from .models import Project, ProjectVersion, AccessRequest, FileHistory, PushChangeType -from ..app import DateTimeWithZ +from ..app import DateTimeWithZ, ma from ..auth.models import User @@ -293,7 +292,8 @@ class Meta: class AdminProjectSchema(ma.SQLAlchemyAutoSchema): id = fields.UUID(attribute="Project.id") name = fields.Str(attribute="Project.name") - namespace = fields.Method("_workspace_name") + workspace = fields.Method("_workspace_name") + workspace_id = fields.Int(attribute="Project.workspace_id") version = fields.Function( lambda obj: ProjectVersion.to_v_name(obj.Project.latest_version) ) diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index 29309241..4debb255 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -16,7 +16,7 @@ from result import Err, Ok, Result from .storage import ProjectStorage, FileNotFound, InitializationError -from ... import db +from ...app import db from ..utils import ( generate_checksum, is_versioned_file, diff --git a/server/mergin/sync/tasks.py b/server/mergin/sync/tasks.py index cfb1f645..ac9fdb1c 100644 --- a/server/mergin/sync/tasks.py +++ b/server/mergin/sync/tasks.py @@ -13,7 +13,7 @@ from .storages.disk import move_to_tmp from .config import Configuration from ..celery import celery -from .. import db +from ..app import db @celery.task diff --git a/server/mergin/sync/workspace.py b/server/mergin/sync/workspace.py index 7781f7be..5b5ac84e 100644 --- a/server/mergin/sync/workspace.py +++ b/server/mergin/sync/workspace.py @@ -18,7 +18,7 @@ ) from .permissions import projects_query, ProjectPermissions from .public_api_controller import parse_project_access_update_request -from .. import db +from ..app import db from ..auth.models import User from ..config import Configuration from .interfaces import AbstractWorkspace, WorkspaceHandler @@ -269,16 +269,17 @@ def monthly_contributors_count(): .count() ) - def projects_query(self, name=None, workspace=None): + def projects_query(self, like: str = None): ws = self.factory_method() query = db.session.query( - Project, literal(ws.name).label("workspace_name") + Project, + literal(ws.name).label("workspace_name"), ).filter(Project.storage_params.isnot(None)) - if name: - query = query.filter(Project.name.ilike(f"%{name}%")) - if workspace: - query = query.filter(literal(ws.name).ilike(f"%{workspace}%")) + if like: + query = query.filter( + Project.name.ilike(f"%{like}%") | literal(ws.name).ilike(f"%{like}%") + ) return query @staticmethod diff --git a/server/mergin/tests/fixtures.py b/server/mergin/tests/fixtures.py index bab7646e..6b83ecf9 100644 --- a/server/mergin/tests/fixtures.py +++ b/server/mergin/tests/fixtures.py @@ -12,7 +12,7 @@ from pygeodiff import GeoDiff import pytest -from .. import db, create_app +from ..app import db, create_app from ..sync.models import Project, ProjectVersion from ..stats.app import register from ..stats.models import MerginInfo diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index c4a12103..41f1065d 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -12,7 +12,7 @@ from ..auth.models import User, UserProfile, LoginHistory from ..auth.tasks import anonymize_removed_users -from .. import db +from ..app import db from ..sync.models import Project from . import ( test_workspace_id, @@ -412,6 +412,25 @@ def test_api_user_profile(client): def test_update_user(client): login_as_admin(client) user = User.query.filter_by(username="mergin").first() + data = {"active": True, "is_admin": True} + resp = client.patch( + url_for("/.mergin_auth_controller_update_user", username=user.username), + data=json.dumps(data), + headers=json_headers, + ) + assert resp.status_code == 200 + assert user.active + assert user.is_admin + + client.application.config["ENABLE_SUPERADMIN_ASSIGNMENT"] = False + data = {"active": False, "is_admin": False} + resp = client.patch( + url_for("/.mergin_auth_controller_update_user", username=user.username), + data=json.dumps(data), + headers=json_headers, + ) + assert resp.status_code == 400 + assert user.active data = {"active": False} resp = client.patch( url_for("/.mergin_auth_controller_update_user", username=user.username), @@ -421,6 +440,7 @@ def test_update_user(client): assert resp.status_code == 200 assert not user.active + client.application.config["ENABLE_SUPERADMIN_ASSIGNMENT"] = True user.is_admin = False db.session.add(user) db.session.commit() @@ -694,27 +714,27 @@ def test_paginate_users(client): url = "/app/admin/users?page=1&per_page=10" # get 5 users (default + 5 new added - 1 deleted & inactive) resp = client.get(url) - list_of_usernames = [user["username"] for user in resp.json["users"]] - assert resp.json["total"] == 5 - assert resp.json["users"][0]["username"] == "mergin" + list_of_usernames = [user["username"] for user in resp.json["items"]] + assert resp.json["count"] == 5 + assert resp.json["items"][0]["username"] == "mergin" assert user_inactive.username in list_of_usernames assert deleted_active.username in list_of_usernames assert deleted_inactive.username not in list_of_usernames # order by username resp = client.get(url + "&order_by=username") - assert resp.json["total"] == 5 - assert resp.json["users"][0]["username"] == "alice" + assert resp.json["count"] == 5 + assert resp.json["items"][0]["username"] == "alice" # exact match with username resp = client.get(url + "&like=bob") - assert resp.json["total"] == 1 - assert resp.json["users"][0]["username"] == "bob" + assert resp.json["count"] == 1 + assert resp.json["items"][0]["username"] == "bob" # ilike search with email resp = client.get(url + "&like=@mergin.com") - assert resp.json["total"] == 5 + assert resp.json["count"] == 5 # exact search by email resp = client.get(url + "&like=alice@mergin.com") - assert resp.json["total"] == 1 - assert resp.json["users"][0]["username"] == "alice" + assert resp.json["count"] == 1 + assert resp.json["items"][0]["username"] == "alice" # invalid paging assert client.get("/app/admin/users?page=2&per_page=10").status_code == 404 diff --git a/server/mergin/tests/test_celery.py b/server/mergin/tests/test_celery.py index cc88e2d3..09b118c6 100644 --- a/server/mergin/tests/test_celery.py +++ b/server/mergin/tests/test_celery.py @@ -8,7 +8,7 @@ from flask_mail import Mail from unittest.mock import patch -from .. import db +from ..app import db from ..config import Configuration from ..sync.models import Project, AccessRequest, ProjectVersion from ..celery import send_email_async diff --git a/server/mergin/tests/test_db_hooks.py b/server/mergin/tests/test_db_hooks.py index 0f16710d..16e7d1e3 100644 --- a/server/mergin/tests/test_db_hooks.py +++ b/server/mergin/tests/test_db_hooks.py @@ -20,7 +20,7 @@ ) from ..sync.files import UploadChanges from ..auth.models import User -from .. import db +from ..app import db from . import DEFAULT_USER from .utils import add_user, create_project, create_workspace, cleanup diff --git a/server/mergin/tests/test_file_restore.py b/server/mergin/tests/test_file_restore.py index f189c9e4..278837d3 100644 --- a/server/mergin/tests/test_file_restore.py +++ b/server/mergin/tests/test_file_restore.py @@ -6,7 +6,7 @@ import shutil from sqlalchemy.orm.attributes import flag_modified -from .. import db +from ..app import db from ..auth.models import User from ..sync.models import ProjectVersion, Project, GeodiffActionHistory from . import test_project_dir, TMP_DIR diff --git a/server/mergin/tests/test_middleware.py b/server/mergin/tests/test_middleware.py new file mode 100644 index 00000000..09f4bf05 --- /dev/null +++ b/server/mergin/tests/test_middleware.py @@ -0,0 +1,33 @@ +# Copyright (C) Lutra Consulting Limited +# +# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + +import gevent +import pytest + +from ..app import create_simple_app, GeventTimeoutMiddleware +from ..config import Configuration + + +@pytest.mark.parametrize("use_middleware", [True, False]) +def test_use_middleware(use_middleware): + """Test using middleware""" + Configuration.GEVENT_WORKER = use_middleware + Configuration.GEVENT_REQUEST_TIMEOUT = 1 + application = create_simple_app() + + def ping(): + gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1) + return "pong" + + application.add_url_rule("/test", "ping", ping) + app_context = application.app_context() + app_context.push() + + assert isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware + # in case of gevent, dummy endpoint it set to time out + assert ( + application.test_client().get("/test").status_code == 504 + if use_middleware + else 200 + ) diff --git a/server/mergin/tests/test_permissions.py b/server/mergin/tests/test_permissions.py index 0e986de3..cfabbdc2 100644 --- a/server/mergin/tests/test_permissions.py +++ b/server/mergin/tests/test_permissions.py @@ -8,7 +8,7 @@ from ..sync.permissions import require_project, ProjectPermissions from ..sync.models import ProjectRole from ..auth.models import User -from .. import db +from ..app import db from ..config import Configuration from .utils import add_user, create_project, create_workspace diff --git a/server/mergin/tests/test_private_project_api.py b/server/mergin/tests/test_private_project_api.py index 9bd66b7b..d8c3145c 100644 --- a/server/mergin/tests/test_private_project_api.py +++ b/server/mergin/tests/test_private_project_api.py @@ -6,15 +6,19 @@ import json import os -import pytest from flask import url_for -from .. import db +from ..app import db from ..sync.models import AccessRequest, Project, ProjectRole, RequestStatus from ..auth.models import User from ..config import Configuration from . import json_headers -from .utils import add_user, login, create_project, create_workspace +from .utils import ( + add_user, + login, + create_project, + create_workspace, +) def test_project_unsubscribe(client, diff_project): @@ -412,7 +416,7 @@ def test_restore_project(client, diff_project): # tests listing resp = client.get("/app/admin/projects?page=1&per_page=10") assert resp.json["count"] == 1 - assert resp.json["projects"][0]["removed_at"] + assert resp.json["items"][0]["removed_at"] diff_project.workspace.active = True db.session.commit() @@ -432,7 +436,7 @@ def test_admin_project_list(client): assert resp.status_code == 200 assert resp.json.get("count") == 1 # mark as inactive - p = Project.query.get(resp.json["projects"][0]["id"]) + p = Project.query.get(resp.json["items"][0]["id"]) p.removed_at = datetime.datetime.utcnow() p.removed_by = user.id db.session.commit() @@ -445,7 +449,7 @@ def test_admin_project_list(client): resp = client.get("/app/admin/projects?page=1&per_page=10&order_params=name ASC") assert resp.status_code == 200 assert resp.json.get("count") == 15 - projects = resp.json.get("projects") + projects = resp.json.get("items") assert len(projects) == 10 assert "foo5" in projects[9]["name"] assert "v0" == projects[9]["version"] @@ -454,30 +458,30 @@ def test_admin_project_list(client): "/app/admin/projects?page=1&per_page=10&order_params=removed_at ASC" ) assert resp.status_code == 200 - assert resp.json["projects"][0]["name"] == p.name + assert resp.json["items"][0]["name"] == p.name resp = client.get( "/app/admin/projects?page=1&per_page=15&order_params=created DESC" ) - assert resp.json["projects"][0]["name"] == "foo13" + assert resp.json["items"][0]["name"] == "foo13" - resp = client.get("/app/admin/projects?page=1&per_page=15&name=12") - assert len(resp.json["projects"]) == 1 - assert resp.json["projects"][0]["name"] == "foo12" + resp = client.get("/app/admin/projects?page=1&per_page=15&like=12") + assert len(resp.json["items"]) == 1 + assert resp.json["items"][0]["name"] == "foo12" - resp = client.get("/app/admin/projects?page=1&per_page=15&name=foo") - assert len(resp.json["projects"]) == 14 + resp = client.get("/app/admin/projects?page=1&per_page=15&like=foo") + assert len(resp.json["items"]) == 14 - resp = client.get("/app/admin/projects?page=1&per_page=15&workspace=invalid") - assert len(resp.json["projects"]) == 0 + resp = client.get("/app/admin/projects?page=1&per_page=15&like=invalid") + assert len(resp.json["items"]) == 0 - resp = client.get("/app/admin/projects?page=1&per_page=15&workspace=mergin") - assert len(resp.json["projects"]) == 15 + resp = client.get("/app/admin/projects?page=1&per_page=15&like=mergin") + assert len(resp.json["items"]) == 15 # delete project permanently p.delete() - resp = client.get("/app/admin/projects?page=1&per_page=15&workspace=mergin") - assert len(resp.json["projects"]) == 14 + resp = client.get("/app/admin/projects?page=1&per_page=15&like=mergin") + assert len(resp.json["items"]) == 14 def test_get_project_access(client): diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 74421147..4371ab8b 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -18,12 +18,13 @@ import re from flask_login import current_user +from unittest.mock import patch from pygeodiff import GeoDiff from flask import url_for, current_app import tempfile from sqlalchemy import desc -from .. import db +from ..app import db from ..sync.models import ( Project, Upload, @@ -1792,6 +1793,8 @@ def test_clone_project(client, data, username, expected): # cleanup shutil.rmtree(project.storage.project_dir) + Configuration.GLOBAL_STORAGE = 104857600 + def test_optimize_storage(app, client, diff_project): """Test optimize storage for geopackages which could be restored from diffs @@ -2480,3 +2483,14 @@ def test_cache_files_ids(client): fp = ProjectFilePath.query.filter_by(project_id=project.id, path=filename).first() fh = FileHistory.query.filter_by(file_path_id=fp.id).first() assert project.latest_project_files.file_history_ids == [fh.id] + + +def test_signals(client): + workspace = create_workspace() + user = User.query.filter(User.username == "mergin").first() + project = create_project("test-project", workspace, user) + with patch( + "mergin.sync.public_api_controller.push_finished.send" + ) as push_finished_mock: + upload_file_to_project(project, "test.txt", client) + push_finished_mock.assert_called_once() diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 4a2e1547..4bdb65a2 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -from .. import db +from ..app import db from mergin.sync.models import Project from tests import test_project, test_workspace_id diff --git a/server/mergin/tests/test_statistics.py b/server/mergin/tests/test_statistics.py index f076d882..2c8e930c 100644 --- a/server/mergin/tests/test_statistics.py +++ b/server/mergin/tests/test_statistics.py @@ -6,7 +6,7 @@ from unittest.mock import patch import requests -from .. import db +from ..app import db from ..stats.tasks import send_statistics from ..stats.models import MerginInfo from .utils import Response diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index a18c6f2c..6141e573 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -10,7 +10,7 @@ from sqlalchemy import desc from unittest.mock import MagicMock -from .. import db +from ..app import db from ..sync.utils import parse_gpkgb_header_size, gpkg_wkb_to_wkt, is_name_allowed from ..auth.models import LoginHistory, User from . import json_headers diff --git a/server/mergin/tests/test_workspace.py b/server/mergin/tests/test_workspace.py index e273d2f1..a10eadb9 100644 --- a/server/mergin/tests/test_workspace.py +++ b/server/mergin/tests/test_workspace.py @@ -6,7 +6,7 @@ from sqlalchemy import null -from .. import db +from ..app import db from ..config import Configuration from ..sync.models import FileHistory, ProjectVersion, PushChangeType, ProjectFilePath from ..sync.workspace import GlobalWorkspaceHandler diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index a9743920..debc096e 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -4,6 +4,7 @@ import json import shutil +from typing import Tuple import pysqlite3 import uuid import math @@ -21,7 +22,7 @@ from ..sync.models import Project, ProjectAccess, ProjectVersion, FileHistory from ..sync.files import UploadChanges, ChangesSchema from ..sync.workspace import GlobalWorkspace -from .. import db +from ..app import db from . import json_headers, DEFAULT_USER, test_project, test_project_dir, TMP_DIR CHUNK_SIZE = 1024 @@ -79,7 +80,7 @@ def create_project(name, workspace, user, **kwargs): p = Project(**project_params, **kwargs) p.updated = datetime.utcnow() - db.session.add(p) + db.session.flush() public = kwargs.get("public", False) pa = ProjectAccess(p, public) @@ -213,10 +214,7 @@ def file_info(project_dir, path, chunk_size=1024): } -def upload_file_to_project(project, filename, client): - """Add test file to project - start, upload and finish push process""" - file = os.path.join(test_project_dir, filename) - assert os.path.exists(file) +def mock_changes_data(project, filename) -> dict: changes = { "added": [file_info(test_project_dir, filename)], "updated": [], @@ -226,13 +224,33 @@ def upload_file_to_project(project, filename, client): "version": ProjectVersion.to_v_name(project.latest_version), "changes": changes, } + return data + + +def push_file_start( + project: Project, filename: str, client, mocked_changes_data=None +) -> dict: + """ + Initiate the process of pushing a file to a project by calling /push endpoint. + """ + file = os.path.join(test_project_dir, filename) + assert os.path.exists(file) + data = mocked_changes_data or mock_changes_data(project, filename) resp = client.post( f"/v1/project/push/{project.workspace.name}/{project.name}", data=json.dumps(data, cls=DateTimeEncoder).encode("utf-8"), headers=json_headers, ) + return resp + + +def upload_file_to_project(project: Project, filename: str, client) -> dict: + """Add test file to project - start, upload and finish push process""" + file = os.path.join(test_project_dir, filename) + data = mock_changes_data(project, filename) + changes = data.get("changes") + resp = push_file_start(project, filename, client, data) upload_id = resp.json["transaction"] - changes = data["changes"] file_meta = changes["added"][0] for chunk_id in file_meta["chunks"]: url = f"/v1/project/push/chunk/{upload_id}/{chunk_id}" @@ -241,7 +259,10 @@ def upload_file_to_project(project, filename, client): client.post( url, data=f_data, headers={"Content-Type": "application/octet-stream"} ) - assert client.post(f"/v1/project/push/finish/{upload_id}").status_code == 200 + push_finish = client.post(f"/v1/project/push/finish/{upload_id}") + assert resp.status_code == 200 + assert push_finish.status_code == 200 + return push_finish def gpkgs_are_equal(file1, file2): diff --git a/web-app/.eslintrc.cjs b/web-app/.eslintrc.cjs index c3e9d504..9689d42e 100644 --- a/web-app/.eslintrc.cjs +++ b/web-app/.eslintrc.cjs @@ -1,15 +1,12 @@ module.exports = { root: true, env: { - es2021: true, - browser: true + es2022: true, + browser: true, + node: true }, - extends: [ - 'plugin:vue/essential', - '@vue/standard', - 'plugin:prettier/recommended' - ], - plugins: ['@typescript-eslint'], + extends: ['plugin:vue/vue3-essential', 'plugin:prettier/recommended'], + plugins: ['@typescript-eslint', 'eslint-plugin-import'], rules: { 'prettier/prettier': 'warn', '@typescript-eslint/no-unused-vars': [ diff --git a/web-app/README.md b/web-app/README.md index 053a0e40..76f3d62f 100644 --- a/web-app/README.md +++ b/web-app/README.md @@ -11,18 +11,17 @@ Monorepo for mergin frontend stuff: * monorepo consists of root directory with own package.json and individual workspaces in `packages` directory * root package.json defines common devDependencies and it cannot hold any non-development dependency * individual workspace package.json defines dependencies used in that particular workspace package -* frontend applications (e.g. app) is also a workspace package (@mergin/app) held in directory [./packages/app](./packages/app). +* frontend application (e.g. app) is also a workspace package (@mergin/app) held in directory [./packages/app](./packages/app). Application constist of several packages in `packages` directory: - @mergin/lib - Shared library for common features -- @mergin/lib-vue2 - Shared library for common features with vuetify and vue 2.7 - @mergin/admin-lib - Shared library for admin - @mergin/app - Web appliacation" - @mergin/admin-app - Web application for administration Library packages with *-lib* name are containing shared code for *-app* applications. -Web application *@mergin/app* is using shared library *@mergin/lib*. Web application for administration *admin-app* is using shared libraries with vue 2.7 *@mergin/admin-lib* and *@mergin/lib-vue2*. +Web application *@mergin/app* is using shared library *@mergin/lib*. Web application for administration *admin-app* is using shared library *@mergin/admin-lib*. For details about development follow instructions in [development guide](../development.md). \ No newline at end of file diff --git a/web-app/package.json b/web-app/package.json index 50771c7c..869ef85b 100755 --- a/web-app/package.json +++ b/web-app/package.json @@ -6,7 +6,6 @@ "packages/admin-lib", "packages/app", "packages/admin-app", - "packages/lib-vue2", "packages/*" ], "scripts": { @@ -25,58 +24,50 @@ "build:dev": "yarn workspace @mergin/app build:dev", "build:admin": "yarn workspace @mergin/admin-app build", "build:admin:dev": "yarn workspace @mergin/admin-app build:dev", - "clean:libs": "yarn workspace @mergin/lib clean && yarn workspace @mergin/admin-lib clean && yarn workspace @mergin/lib-vue2 clean", + "clean:libs": "yarn workspace @mergin/lib clean && yarn workspace @mergin/admin-lib clean", "clean:libs:noadmin": "yarn workspace @mergin/lib clean", - "i18n:report": "yarn workspace @mergin/app 18n:report", - "link:dependencies": "yarn link:register && yarn link @mergin/lib && yarn link @mergin/admin-lib && yarn link @mergin/lib-vue2", + "link:dependencies": "yarn link:register && yarn link @mergin/lib && yarn link @mergin/admin-lib", "link:register": "yarn workspaces run link:register", "link:unregister": "yarn workspaces run link:unregister", "types:libs": "yarn workspaces run build:types", "types:lib": "yarn workspace @mergin/lib build:types", - "types:lib-vue2": "yarn workspace @mergin/lib-vue2 build:types", "types:admin-lib": "yarn workspace @mergin/admin-lib build:types", "watch:lib": "yarn workspace @mergin/lib build:lib:watch", - "watch:lib-vue2": "yarn workspace @mergin/lib-vue2 build:lib:watch", "watch:lib:types": "yarn workspace @mergin/lib build:types:watch", - "watch:lib-vue2:types": "yarn workspace @mergin/lib-vue2 build:types:watch", "watch:admin-lib": "yarn workspace @mergin/admin-lib build:lib:watch", "watch:admin-lib:types": "yarn workspace @mergin/admin-lib build:types:watch", "lint:all": "yarn workspaces run lint", "lint:no-legacy": "yarn workspace @mergin/lib lint && yarn workspace @mergin/app lint" }, "devDependencies": { - "@babel/core": "^7.17.9", "@esbuild-plugins/node-globals-polyfill": "^0.2.3", "@esbuild-plugins/node-modules-polyfill": "^0.2.2", - "@typescript-eslint/eslint-plugin": "^5.61.0", - "@typescript-eslint/parser": "^5.61.0", - "@vitejs/plugin-vue": "^4.6.2", - "@vitejs/plugin-vue2": "^2.2.0", - "@vue/eslint-config-standard": "^8.0.1", - "@vue/eslint-config-typescript": "^11.0.2", + "@typescript-eslint/eslint-plugin": "^8.11.0", + "@typescript-eslint/parser": "^8.11.0", + "@vitejs/plugin-vue": "^5.1.4", + "@vue/eslint-config-typescript": "^13.0.0", "agent-base": "^6.0.2", "autoprefixer": "^10.4.14", "concurrently": "^8.0.1", - "eslint": "^8.37.0", + "eslint": "^8.57.1", "eslint-config-prettier": "^8.8.0", - "eslint-plugin-import": "^2.27.5", + "eslint-plugin-import": "^2.31.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-vue": "^9.10.0", + "eslint-plugin-vue": "^9.29.1", "npm-run-all": "^4.1.5", - "postcss": "^8.4.21", + "postcss": "^8.4.47", "prettier": "2.6.0", "purgecss": "^5.0.0", "rimraf": "^3.0.2", - "rollup-plugin-copy": "^3.4.0", + "sass": "^1.69.5", "tsc-alias": "^1.8.6", - "typescript": "^5.0.4", - "unplugin-vue-components": "^0.25.1", - "vite": "^4.5.3", - "vite-plugin-static-copy": "^0.17.0", - "vite-plugin-vuetify": "^1.0.2", - "vue-tsc": "^1.6.5" + "typescript": "^5.6.3", + "unplugin-vue-components": "^0.27.1", + "vite": "^5.4.10", + "vite-plugin-static-copy": "^2.0.0", + "vue-tsc": "^2.1.6" }, "engines": { "node": ">=18" diff --git a/web-app/packages/admin-app/.eslintrc.cjs b/web-app/packages/admin-app/.eslintrc.cjs index 51dfd9e4..9a294e26 100644 --- a/web-app/packages/admin-app/.eslintrc.cjs +++ b/web-app/packages/admin-app/.eslintrc.cjs @@ -1,11 +1,9 @@ module.exports = { - env: { - node: true - }, extends: [ 'plugin:vue/essential', 'eslint:recommended', - '@vue/typescript/recommended' + '@vue/typescript/recommended', + '../../.eslintrc.cjs' ], parser: 'vue-eslint-parser', parserOptions: { @@ -16,6 +14,9 @@ module.exports = { plugins: ['@typescript-eslint'], rules: { 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', - 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off' + 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off', + // Vue 3 opt https://v3-migration.vuejs.org/breaking-changes/key-attribute.html#with-template-v-for + 'vue/no-v-for-template-key-on-child': 'error', + 'vue/no-v-for-template-key': 'off', } } diff --git a/web-app/packages/admin-app/components.d.ts b/web-app/packages/admin-app/components.d.ts index 277cd1f0..6bcacfd1 100644 --- a/web-app/packages/admin-app/components.d.ts +++ b/web-app/packages/admin-app/components.d.ts @@ -1,12 +1,13 @@ /* eslint-disable */ -/* prettier-ignore */ // @ts-nocheck // Generated by unplugin-vue-components // Read more: https://github.com/vuejs/core/pull/3399 export {} +/* prettier-ignore */ declare module 'vue' { export interface GlobalComponents { + PDivider: typeof import('primevue/divider')['default'] RouterLink: typeof import('vue-router')['RouterLink'] RouterView: typeof import('vue-router')['RouterView'] VApp: typeof import('vuetify/lib')['VApp'] diff --git a/web-app/packages/admin-app/index.html b/web-app/packages/admin-app/index.html index e4efe3b2..afdb9b39 100644 --- a/web-app/packages/admin-app/index.html +++ b/web-app/packages/admin-app/index.html @@ -11,7 +11,7 @@ - Mergin Maps + Mergin Maps Admin Panel