diff --git a/lizmap/definitions/lizmap_cloud.py b/lizmap/definitions/lizmap_cloud.py new file mode 100644 index 00000000..d6ea29a8 --- /dev/null +++ b/lizmap/definitions/lizmap_cloud.py @@ -0,0 +1,10 @@ +__copyright__ = 'Copyright 2023, 3Liz' +__license__ = 'GPL version 3' +__email__ = 'info@3liz.org' + +CLOUD_DOMAIN = 'lizmap.com' +CLOUD_NAME = 'Lizmap Cloud' +CLOUD_MAX_PARENT_FOLDER = 2 # TODO Check COG, is-it 3 ? + +CLOUD_ONLINE_URL = 'https://docs.lizmap.cloud' +CLOUD_ONLINE_LANGUAGES = ('en', 'fr') diff --git a/lizmap/definitions/online_help.py b/lizmap/definitions/online_help.py index acf8aa7a..92a08d73 100644 --- a/lizmap/definitions/online_help.py +++ b/lizmap/definitions/online_help.py @@ -7,13 +7,15 @@ from qgis.core import QgsSettings from qgis.PyQt.QtCore import QLocale, QUrl +from lizmap.definitions.lizmap_cloud import ( + CLOUD_ONLINE_LANGUAGES, + CLOUD_ONLINE_URL, +) + DOMAIN = 'https://docs.lizmap.com' VERSION = 'current' ONLINE_HELP_LANGUAGES = ('en', 'es', 'it', 'ja', 'pt', 'fi', 'fr') -CLOUD = 'https://docs.lizmap.cloud' -CLOUD_HELP_LANGUAGES = ('en', 'fr') - def current_locale() -> str: """ Get the main language, with 2 characters only. """ @@ -25,9 +27,9 @@ def current_locale() -> str: def online_cloud_help(page: str = '') -> QUrl: """ Online help URL according to locale and version. """ locale = current_locale() - if locale not in CLOUD_HELP_LANGUAGES: + if locale not in CLOUD_ONLINE_LANGUAGES: locale = 'en' - return QUrl(f"{CLOUD}/{locale}/{page}") + return QUrl(f"{CLOUD_ONLINE_URL}/{locale}/{page}") def online_lwc_help(page: str = '', version=VERSION) -> QUrl: diff --git a/lizmap/definitions/qgis_settings.py b/lizmap/definitions/qgis_settings.py new file mode 100644 index 00000000..e64fdb3c --- /dev/null +++ b/lizmap/definitions/qgis_settings.py @@ -0,0 +1,26 @@ +"""Definitions for QgsSettings.""" + +# TODO, use the settings API from QGIS 3.30 etc +# Mail QGIS-Dev 24/10/2023 + +__copyright__ = 'Copyright 2023, 3Liz' +__license__ = 'GPL version 3' +__email__ = 'info@3liz.org' + +KEY = 'lizmap' + + +class Settings: + + @classmethod + def key(cls, key): + return KEY + '/' + key + + PreventEcw = 'prevent_ecw' + PreventPgAuthDb = 'prevent_pg_auth_db' + PreventPgService = 'prevent_pg_service' + ForcePgUserPass = 'force_pg_user_password' + PreventDrive = 'prevent_drive' + AllowParentFolder = 'allow_parent_folder' + NumberParentFolder = 'number_parent_folder' + BeginnerMode = 'beginner_mode' diff --git a/lizmap/definitions/warnings.py b/lizmap/definitions/warnings.py deleted file mode 100644 index b137d81e..00000000 --- a/lizmap/definitions/warnings.py +++ /dev/null @@ -1,15 +0,0 @@ -__copyright__ = 'Copyright 2022, 3Liz' -__license__ = 'GPL version 3' -__email__ = 'info@3liz.org' - - -from enum import Enum, unique - - -@unique -class Warnings(Enum): - OgcNotValid = 'ogc_not_valid' - UseLayerIdAsName = 'use_layer_id_as_name' - SaasLizmapCloud = 'saas_lizmap_cloud_invalid' - InvalidFieldType = 'invalid_field_type' - DuplicatedLayersWithFilters = 'duplicated_layers_with_filters' diff --git a/lizmap/dialogs/download_from_url.py b/lizmap/dialogs/download_from_url.py new file mode 100644 index 00000000..cc2990b9 --- /dev/null +++ b/lizmap/dialogs/download_from_url.py @@ -0,0 +1,380 @@ +__copyright__ = 'Copyright 2023, 3Liz' +__license__ = 'GPL version 3' +__email__ = 'info@3liz.org' + +import json +import logging +import tempfile + +from enum import Enum +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +from qgis.core import ( + Qgis, + QgsApplication, + QgsAuthMethodConfig, + QgsBlockingNetworkRequest, + QgsFileDownloader, + QgsProject, +) +from qgis.gui import QgsAuthConfigSelect +from qgis.PyQt.QtCore import QEventLoop, Qt, QUrl +from qgis.PyQt.QtGui import QDesktopServices, QIcon +from qgis.PyQt.QtNetwork import QNetworkRequest +from qgis.PyQt.QtWidgets import ( + QDialog, + QDialogButtonBox, + QLabel, + QLineEdit, + QPushButton, + QVBoxLayout, +) +from qgis.utils import OverrideCursor, iface + + +class UriType(Enum): + Finder = 1 + Nautilus = 2 + Dolphin = 3 + WinScp = 4 + Gvfs = 5 + + +LOGGER = logging.getLogger('Lizmap') + + +class DownloadProject(QDialog): + + def __init__(self, auth_id: str = None): + # noinspection PyArgumentList + QDialog.__init__(self) + self.setMinimumWidth(800) + layout = QVBoxLayout() + + self.auth = QgsAuthConfigSelect(self) + if not auth_id: + auth_id = "x70445w" + + if auth_id: + self.auth.setConfigId(auth_id) + self.project_url = QLineEdit(self) + self.project_url.textChanged.connect(self.update_server) + self.server_url = QLineEdit(self) + + debug_project = "https://demo.lizmap.com/lizmap/index.php/view/map?repository=features&project=observations" + debug_project = "http://localhost:9023/index.php/view/map?repository=tests&project=fire_hydrant_actions" + + self.project_url.setPlaceholderText(debug_project) + + self.update_server() + + # noinspection PyArgumentList + layout.addWidget(QLabel("Authentification perso")) + # noinspection PyArgumentList + layout.addWidget(self.auth) + + # noinspection PyArgumentList + layout.addWidget(QLabel("Project URL")) + # noinspection PyArgumentList + layout.addWidget(self.project_url) + + # noinspection PyArgumentList + layout.addWidget(QLabel("Server URL")) + # noinspection PyArgumentList + layout.addWidget(self.server_url) + + temp_dir = tempfile.TemporaryDirectory() + + # noinspection PyArgumentList + layout.addWidget(QLabel("Destination")) + self.directory = QLineEdit(self) + self.directory.setPlaceholderText(temp_dir.name) + # noinspection PyArgumentList + layout.addWidget(self.directory) + + # noinspection PyArgumentList + self.webdav_label_win = QLabel("WebDav URI (Mac, Finder, Curl)") + self.webdav_label_win.setVisible(False) + layout.addWidget(self.webdav_label_win) + self.webdav_uri_win = QLineEdit(self) + self.webdav_uri_win.setReadOnly(True) + self.webdav_uri_win.setVisible(False) + # noinspection PyArgumentList + layout.addWidget(self.webdav_uri_win) + + # noinspection PyArgumentList + self.webdav_label_gnome = QLabel("WebDav URI (Nautilus)") + self.webdav_label_gnome.setVisible(False) + layout.addWidget(self.webdav_label_gnome) + self.webdav_uri_gnome = QLineEdit(self) + self.webdav_uri_gnome.setReadOnly(True) + self.webdav_uri_gnome.setVisible(False) + # noinspection PyArgumentList + layout.addWidget(self.webdav_uri_gnome) + + # noinspection PyArgumentList + self.webdav_label_gvfs = QLabel("WebDav URI (GVFS)") + self.webdav_label_gvfs.setVisible(False) + layout.addWidget(self.webdav_label_gvfs) + self.webdav_uri_gvfs = QLineEdit(self) + self.webdav_uri_gvfs.setReadOnly(True) + self.webdav_uri_gvfs.setVisible(False) + # noinspection PyArgumentList + layout.addWidget(self.webdav_uri_gvfs) + + # noinspection PyArgumentList + self.webdav_label_kde = QLabel("WebDav URI (Dolphin KDE)") + self.webdav_label_kde.setVisible(False) + layout.addWidget(self.webdav_label_kde) + self.webdav_uri_kde = QLineEdit(self) + self.webdav_uri_kde.setReadOnly(True) + self.webdav_uri_kde.setVisible(False) + # noinspection PyArgumentList + layout.addWidget(self.webdav_uri_kde) + + # noinspection PyArgumentList + self.webdav_label_winscp = QLabel("WebDav URI (WinSCP, CyberDuck)") + self.webdav_label_winscp.setVisible(False) + layout.addWidget(self.webdav_label_winscp) + self.webdav_uri_winscp = QLineEdit(self) + self.webdav_uri_winscp.setReadOnly(True) + self.webdav_uri_winscp.setVisible(False) + # noinspection PyArgumentList + layout.addWidget(self.webdav_uri_winscp) + + self.button_file_browser = QPushButton("Navigateur de fichier") + self.button_file_browser.setVisible(False) + self.button_file_browser.clicked.connect(self.open_file_browser) + self.button_file_browser.setIcon(QIcon(":/images/themes/default/mIconFolderOpen.svg")) + layout.addWidget(self.button_file_browser) + + self.button_box = QDialogButtonBox() + self.button_box.setStandardButtons(QDialogButtonBox.Cancel | QDialogButtonBox.Ok) + # noinspection PyArgumentList + layout.addWidget(self.button_box) + + self.setLayout(layout) + + accept_button = self.button_box.button(QDialogButtonBox.Ok) + accept_button.clicked.connect(self.download) + cancel_button = self.button_box.button(QDialogButtonBox.Cancel) + cancel_button.clicked.connect(self.reject) + + self.destination = None + + def update_server(self): + """ Update base URL of the server. """ + server_metadata = self.project_url.text() + if not server_metadata: + server_metadata = self.project_url.placeholderText() + + result = server_metadata.split('index.php') + if not result: + return + + self.server_url.setText(result[0]) + + def _uri(self, webdav_url: str, uri_type: UriType) -> str: + """ Different kind of URI. """ + auth_id = self.auth.configId() + conf = QgsAuthMethodConfig() + QgsApplication.authManager().loadAuthenticationConfig(auth_id, conf, True) + if not conf.id(): + return '' + + parsed_result = urlparse(webdav_url) + url = parsed_result.netloc + parsed_result.path + + user = conf.config('username') + _ = conf.config('password') + + if uri_type == UriType.Nautilus: + # https://en.wikipedia.org/wiki/Uniform_Resource_Identifier + # user:password + uri = f'davs://{user}@{url}' + elif uri_type == UriType.Finder: + uri = f'https://{url}' + elif uri_type == UriType.Dolphin: + uri = f'webdav://{url}' + elif uri_type == UriType.Gvfs: + # Remove last trailing slash + escaped = parsed_result.path[0:-1] + escaped = escaped.replace('/', '%2F') + uri = ( + f"/run/user/1000/gvfs/dav:" + f"host={parsed_result.hostname}," + f"ssl=true," + f"user={user}," + f"prefix={escaped}" + ) + else: + uri = url + + return uri + + def update_uri_gui(self, webdav_url: str): + """ Update all URI for the webdav server. """ + # self.webdav_label_win.setVisible(True) + # self.webdav_uri_win.setText(self._uri(webdav_url, UriType.Finder)) + # self.webdav_uri_win.setVisible(True) + + self.webdav_label_gnome.setVisible(True) + self.webdav_uri_gnome.setText(self._uri(webdav_url, UriType.Nautilus)) + self.webdav_uri_gnome.setVisible(True) + + # self.webdav_label_winscp.setVisible(True) + # self.webdav_uri_winscp.setText(self._uri(webdav_url, UriType.WinScp)) + # self.webdav_uri_winscp.setVisible(True) + # + # self.webdav_label_gvfs.setVisible(True) + # self.webdav_uri_gvfs.setText(self._uri(webdav_url, UriType.Gvfs)) + # self.webdav_uri_gvfs.setVisible(True) + # + # self.webdav_label_kde.setVisible(True) + # self.webdav_uri_kde.setText(self._uri(webdav_url, UriType.Dolphin)) + # self.webdav_uri_kde.setVisible(True) + + # Only a single button with GVFS for now + # self.button_file_browser.setVisible(True) + # self.button_file_browser.setToolTip(self._uri(webdav_url, UriType.Gvfs)) + + def open_file_browser(self): + """ Try to open the file browser. """ + QDesktopServices.openUrl(QUrl.fromLocalFile(self.button_file_browser.toolTip())) + + def download(self): + """ Download files. """ + server_metadata = self.server_url.text() + if not server_metadata: + server_metadata = self.server_url.placeholderText() + if not server_metadata.endswith('/'): + server_metadata += '/' + server_metadata += 'index.php/view/app/metadata' + + if not self.auth.configId(): + return + + with OverrideCursor(Qt.WaitCursor): + net_req = QNetworkRequest() + # noinspection PyArgumentList + net_req.setUrl(QUrl(server_metadata)) + net_req.setAttribute(QNetworkRequest.FollowRedirectsAttribute, True) + request = QgsBlockingNetworkRequest() + request.setAuthCfg(self.auth.configId()) + request.get(net_req) + response = request.reply().content() + content = json.loads(response.data().decode('utf-8')) + + if not content: + iface.messageBar().pushMessage( + title='Lizmap', + text='Erreur lors du téléchargement', + level=Qgis.Critical, + duration=5, + ) + self.close() + return + + project_url = self.project_url.text() + if not project_url: + project_url = self.project_url.placeholderText() + + result = urlparse(project_url) + qs = parse_qs(result.query) + repo_id = qs['repository'][0] + project_id = qs['project'][0] + + repository = content['repositories'][repo_id] + + if not content.get('webdav'): + iface.messageBar().pushMessage( + title='Lizmap', + text='Pas de webdav sur le serveur', + level=Qgis.Critical, + duration=5, + ) + return + + webdav_url = content['webdav']['url'] + if not webdav_url.endswith('/'): + webdav_url += '/' + + self.update_uri_gui(webdav_url) + + dav_project_path = content['webdav']['projects_path'] + if dav_project_path: + if not dav_project_path.endswith('/'): + dav_project_path += '/' + webdav_url += dav_project_path + + webdav_url += repository['path'] + if not webdav_url.endswith('/'): + webdav_url += '/' + + webdav_url += project_id + + files = (webdav_url + '.qgs.cfg', webdav_url + '.qgs') + + output = self.directory.text() + if not output: + output = self.directory.placeholderText() + output_dir = Path(output) + if not output_dir.exists(): + output_dir.mkdir() + + self.destination = output_dir + + for file in files: + + if file.endswith('.qgs'): + destination_file = output_dir.joinpath(project_id + '.qgs') + else: + destination_file = output_dir.joinpath(project_id + '.qgs.cfg') + + downloader = QgsFileDownloader( + QUrl(file), str(destination_file), delayStart=True, authcfg=self.auth.configId()) + loop = QEventLoop() + downloader.downloadExited.connect(loop.quit) + downloader.downloadError.connect(self.error) + downloader.downloadCanceled.connect(self.canceled) + downloader.downloadCompleted.connect(self.completed) + downloader.startDownload() + loop.exec_() + + if destination_file.exists() and file.endswith('.qgs'): + QgsProject.instance().read(str(destination_file)) + + # self.close() + + @staticmethod + def error(messages: str): + """Store the messages error""" + iface.messageBar().pushMessage( + title='Lizmap', + text=messages, + level=Qgis.Critical, + duration=5, + ) + return + + @staticmethod + def canceled(): + """Display the status in logger""" + iface.messageBar().pushMessage( + title='Lizmap', + text="Cancelled", + level=Qgis.Warning, + duration=5, + ) + return + + def completed(self): + """Display the status in logger""" + iface.messageBar().pushMessage( + title='Lizmap', + text="Got it {}".format(self.destination, self.destination), + level=Qgis.Success, + duration=10, + ) + return diff --git a/lizmap/dialogs/download_project.py b/lizmap/dialogs/download_project.py new file mode 100644 index 00000000..d5bba4e6 --- /dev/null +++ b/lizmap/dialogs/download_project.py @@ -0,0 +1,76 @@ +__copyright__ = 'Copyright 2023, 3Liz' +__license__ = 'GPL version 3' +__email__ = 'info@3liz.org' + +import logging + +from qgis.PyQt.QtWidgets import ( + QComboBox, + QDialog, + QDialogButtonBox, + QVBoxLayout, +) + +LOGGER = logging.getLogger('Lizmap') + + +class DownloadProject(QDialog): + + def __init__(self, metadata: dict): + # noinspection PyArgumentList + QDialog.__init__(self) + + layout = QVBoxLayout() + + self.metadata = metadata + + self.directory = QComboBox(self) + # noinspection PyArgumentList + layout.addWidget(self.directory) + + for directory in metadata['repositories']: + label = metadata['repositories'][directory]["label"] + self.directory.addItem(f'{label} : {directory}', directory) + + self.project = QComboBox(self) + # noinspection PyArgumentList + layout.addWidget(self.project) + + self.directory.currentIndexChanged.connect(self.update_project) + self.update_project() + + self.button_box = QDialogButtonBox() + self.button_box.setStandardButtons(QDialogButtonBox.Cancel | QDialogButtonBox.Ok) + # noinspection PyArgumentList + layout.addWidget(self.button_box) + + self.setLayout(layout) + + accept_button = self.button_box.button(QDialogButtonBox.Ok) + accept_button.clicked.connect(self.accept) + cancel_button = self.button_box.button(QDialogButtonBox.Cancel) + cancel_button.clicked.connect(self.reject) + + def update_project(self): + directory = self.directory.currentData() + for project in self.metadata['repositories'][directory]['projects']: + label = self.metadata['repositories'][directory]['projects'][project]['title'] + self.project.addItem(f'{label} : {project}', project) + + +if __name__ == '__main__': + """ For manual tests. """ + import json + import sys + + from qgis.PyQt.QtWidgets import QApplication + + from lizmap.qgis_plugin_tools.tools.resources import plugin_test_data_path + + app = QApplication(sys.argv) + with open(plugin_test_data_path('metadata', '10102023.json')) as f: + content = json.load(f) + + dialog = DownloadProject(content) + dialog.show() + sys.exit(app.exec_()) diff --git a/lizmap/dialogs/main.py b/lizmap/dialogs/main.py index 45d5d7a4..7a321e9d 100755 --- a/lizmap/dialogs/main.py +++ b/lizmap/dialogs/main.py @@ -28,13 +28,23 @@ ) from qgis.utils import OverrideCursor, iface +from lizmap.definitions.lizmap_cloud import CLOUD_MAX_PARENT_FOLDER, CLOUD_NAME +from lizmap.definitions.qgis_settings import Settings from lizmap.log_panel import LogPanel from lizmap.project_checker_tools import ( + ALLOW_PARENT_FOLDER, + FORCE_LOCAL_FOLDER, + FORCE_PG_USER_PASS, + PREVENT_AUTH_DB, + PREVENT_ECW, + PREVENT_OTHER_DRIVE, + PREVENT_SERVICE, project_trust_layer_metadata, simplify_provider_side, use_estimated_metadata, ) from lizmap.saas import fix_ssl +from lizmap.widgets.check_project import Checks, TableCheck try: from qgis.PyQt.QtWebKitWidgets import QWebView @@ -51,7 +61,12 @@ from lizmap.qgis_plugin_tools.tools.i18n import tr from lizmap.qgis_plugin_tools.tools.resources import load_ui, resources_path from lizmap.qt_style_sheets import COMPLETE_STYLE_SHEET -from lizmap.tools import format_qgis_version, human_size, qgis_version +from lizmap.tools import ( + format_qgis_version, + human_size, + qgis_version, + relative_path, +) FORM_CLASS = load_ui('ui_lizmap.ui') LOGGER = logging.getLogger("Lizmap") @@ -163,7 +178,7 @@ def __init__(self, parent=None, is_dev_version=True): self.button_use_estimated_md.setIcon(QIcon(":images/themes/default/mIconPostgis.svg")) self.button_trust_project.clicked.connect(self.fix_project_trust) - # self.button_trust_project.setIcon(QIcon(":images/themes/default/mIconPostgis.svg")) + self.button_trust_project.setIcon(QIcon(':/images/themes/default/mIconQgsProjectFile.svg')) self.button_simplify_geom.clicked.connect(self.fix_simplify_geom_provider) self.button_simplify_geom.setIcon(QIcon(":images/themes/default/mIconPostgis.svg")) @@ -217,6 +232,107 @@ def __init__(self, parent=None, is_dev_version=True): ) self.label_file_action.setOpenExternalLinks(True) + self.radio_beginner.setToolTip(tr( + 'If one safeguard is not OK, the Lizmap configuration file is not going to be generated.' + )) + self.radio_normal.setToolTip(tr( + 'If one safeguard is not OK, only a warning will be displayed, not blocking the saving of the Lizmap ' + 'configuration file.' + )) + + self.radio_force_local_folder.setText(FORCE_LOCAL_FOLDER) + self.radio_force_local_folder.setToolTip(tr( + 'Files must be located in {} or in a sub directory.' + ).format(self.project.absolutePath())) + self.radio_allow_parent_folder.setText(ALLOW_PARENT_FOLDER) + self.radio_allow_parent_folder.setToolTip(tr( + 'Files can be located in a parent folder from {}, up to the setting below.' + ).format(self.project.absolutePath())) + + self.safe_other_drive.setText(PREVENT_OTHER_DRIVE) + self.safe_pg_service.setText(PREVENT_SERVICE) + self.safe_pg_auth_db.setText(PREVENT_AUTH_DB) + self.safe_pg_user_password.setText(FORCE_PG_USER_PASS) + self.safe_ecw.setText(PREVENT_ECW) + + # Normal / beginner + self.radio_normal.setChecked( + not QgsSettings().value(Settings.key(Settings.BeginnerMode), type=bool)) + self.radio_beginner.setChecked( + QgsSettings().value(Settings.key(Settings.BeginnerMode), type=bool)) + self.radio_normal.toggled.connect(self.radio_mode_normal_toggled) + self.radio_normal.toggled.connect(self.save_settings) + self.radio_mode_normal_toggled() + + # Parent or subdirectory + self.radio_force_local_folder.setChecked( + not QgsSettings().value(Settings.key(Settings.AllowParentFolder), type=bool)) + self.radio_allow_parent_folder.setChecked( + QgsSettings().value(Settings.key(Settings.AllowParentFolder), type=bool)) + self.radio_allow_parent_folder.toggled.connect(self.radio_parent_folder_toggled) + self.radio_allow_parent_folder.toggled.connect(self.save_settings) + self.radio_parent_folder_toggled() + + # Number + self.safe_number_parent.setValue(QgsSettings().value(Settings.key(Settings.NumberParentFolder), type=int)) + self.safe_number_parent.valueChanged.connect(self.save_settings) + + # Other drive + self.safe_other_drive.setChecked(QgsSettings().value(Settings.key(Settings.PreventDrive), type=bool)) + self.safe_other_drive.toggled.connect(self.save_settings) + self.safe_other_drive.setToolTip(Checks.PreventDrive.description) + + # PG Service + self.safe_pg_service.setChecked(QgsSettings().value(Settings.key(Settings.PreventPgService), type=bool)) + self.safe_pg_service.toggled.connect(self.save_settings) + self.safe_pg_service.setToolTip(Checks.PgService.description) + + # PG Auth DB + self.safe_pg_auth_db.setChecked(QgsSettings().value(Settings.key(Settings.PreventPgAuthDb), type=bool)) + self.safe_pg_auth_db.toggled.connect(self.save_settings) + self.safe_pg_auth_db.setToolTip(Checks.AuthenticationDb.description) + + # User password + self.safe_pg_user_password.setChecked(QgsSettings().value(Settings.key(Settings.ForcePgUserPass), type=bool)) + self.safe_pg_user_password.toggled.connect(self.save_settings) + self.safe_pg_user_password.setToolTip(Checks.PgForceUserPass.description) + + # ECW + self.safe_ecw.setChecked(QgsSettings().value(Settings.key(Settings.PreventEcw), type=bool)) + self.safe_ecw.toggled.connect(self.save_settings) + self.safe_ecw.setToolTip(Checks.PreventEcw.description) + + self.label_safe_lizmap_cloud.setText(tr( + "Some safeguards are overridden by {host}. Even in 'normal' mode, some safeguards are becoming 'blocking' " + "with a {host} instance.").format(host=CLOUD_NAME)) + msg = ( + ''.format( + max_parent=tr("Maximum of parent folder {} : {}").format( + CLOUD_MAX_PARENT_FOLDER, relative_path(CLOUD_MAX_PARENT_FOLDER)), + network=PREVENT_OTHER_DRIVE, + auth_db=PREVENT_AUTH_DB, + user_pass=FORCE_PG_USER_PASS, + ecw=PREVENT_ECW, + ) + ) + self.label_safe_lizmap_cloud.setToolTip(msg) + + self.table_checks.setup() + css_path = resources_path('css', 'log.css') + with open(css_path, encoding='utf8') as f: + css = f.read() + self.html_help.document().setDefaultStyleSheet(css) + + @property + def check_results(self) -> TableCheck: + return self.table_checks + def check_api_key_address(self): """ Check the API key is provided for the address search bar. """ provider = self.liExternalSearch.currentData() @@ -775,6 +891,58 @@ def check_action_file_exists(self) -> bool: self.label_file_action_found.setText("" + tr('Not found') + "") return False + def radio_parent_folder_toggled(self): + """ When the parent allowed folder radio is toggled. """ + parent_allowed = self.radio_allow_parent_folder.isChecked() + widgets = ( + self.label_parent_folder, + self.safe_number_parent, + ) + for widget in widgets: + widget.setEnabled(parent_allowed) + + def radio_mode_normal_toggled(self): + """ When the beginner/normal radio are toggled. """ + is_normal = self.radio_normal.isChecked() + widgets = ( + self.group_file_layer, + self.safe_number_parent, + self.safe_other_drive, + self.safe_pg_service, + self.safe_pg_auth_db, + self.safe_pg_user_password, + self.safe_ecw, + self.label_parent_folder, + ) + for widget in widgets: + widget.setEnabled(is_normal) + widget.setVisible(is_normal) + + def safeguards_to_markdown(self) -> str: + """ Export the list of safeguards to markdown. """ + text = 'List of safeguards :\n' + text += '* Mode : {}\n'.format('normal' if self.radio_normal.isChecked() else 'safe') + text += '* Allow parent folder : {}\n'.format('yes' if self.radio_allow_parent_folder.isChecked() else 'no') + if self.radio_allow_parent_folder.isChecked(): + text += '* Number of parent : {} folder(s)\n'.format(self.safe_number_parent.value()) + text += '* Prevent other drive : {}\n'.format('yes' if self.safe_other_drive.isChecked() else 'no') + text += '* Prevent PG service : {}\n'.format('yes' if self.safe_pg_service.isChecked() else 'no') + text += '* Prevent PG Auth DB : {}\n'.format('yes' if self.safe_pg_auth_db.isChecked() else 'no') + text += '* Force PG user&pass : {}\n'.format('yes' if self.safe_pg_user_password.isChecked() else 'no') + text += '* Prevent ECW : {}\n'.format('yes' if self.safe_ecw.isChecked() else 'no') + return text + + def save_settings(self): + """ Save settings checkboxes. """ + QgsSettings().setValue(Settings.key(Settings.BeginnerMode), not self.radio_normal.isChecked()) + QgsSettings().setValue(Settings.key(Settings.AllowParentFolder), self.radio_allow_parent_folder.isChecked()) + QgsSettings().setValue(Settings.key(Settings.NumberParentFolder), self.safe_number_parent.value()) + QgsSettings().setValue(Settings.key(Settings.PreventDrive), self.safe_other_drive.isChecked()) + QgsSettings().setValue(Settings.key(Settings.PreventPgService), self.safe_pg_service.isChecked()) + QgsSettings().setValue(Settings.key(Settings.PreventPgAuthDb), self.safe_pg_auth_db.isChecked()) + QgsSettings().setValue(Settings.key(Settings.ForcePgUserPass), self.safe_pg_user_password.isChecked()) + QgsSettings().setValue(Settings.key(Settings.PreventEcw), self.safe_ecw.isChecked()) + def allow_navigation(self, allow_navigation: bool, message: str = ''): """ Allow the navigation or not in the UI. """ for i in range(1, self.mOptionsListWidget.count()): diff --git a/lizmap/plugin.py b/lizmap/plugin.py index 0a7295a6..0bf745e5 100755 --- a/lizmap/plugin.py +++ b/lizmap/plugin.py @@ -88,21 +88,21 @@ from lizmap.definitions.filter_by_login import FilterByLoginDefinitions from lizmap.definitions.filter_by_polygon import FilterByPolygonDefinitions from lizmap.definitions.layouts import LayoutsDefinitions +from lizmap.definitions.lizmap_cloud import CLOUD_MAX_PARENT_FOLDER, CLOUD_NAME from lizmap.definitions.locate_by_layer import LocateByLayerDefinitions from lizmap.definitions.online_help import ( MAPPING_INDEX_DOC, online_cloud_help, online_lwc_help, ) +from lizmap.definitions.qgis_settings import Settings from lizmap.definitions.time_manager import TimeManagerDefinitions from lizmap.definitions.tooltip import ToolTipDefinitions -from lizmap.definitions.warnings import Warnings from lizmap.dialogs.dock_html_preview import HtmlPreview from lizmap.dialogs.html_editor import HtmlEditorDialog from lizmap.dialogs.html_maptip import HtmlMapTipDialog from lizmap.dialogs.lizmap_popup import LizmapPopupDialog from lizmap.dialogs.main import LizmapDialog -from lizmap.dialogs.scroll_message_box import ScrollMessageBox from lizmap.dialogs.wizard_group import WizardGroupDialog from lizmap.drag_drop_dataviz_manager import DragDropDatavizManager from lizmap.forms.atlas_edition import AtlasEditionDialog @@ -119,20 +119,22 @@ from lizmap.lizmap_api.config import LizmapConfig from lizmap.ogc_project_validity import OgcProjectValidity from lizmap.project_checker_tools import ( - auto_generated_primary_key_field, + ALLOW_PARENT_FOLDER, + FORCE_LOCAL_FOLDER, + FORCE_PG_USER_PASS, + PREVENT_AUTH_DB, + PREVENT_ECW, + PREVENT_OTHER_DRIVE, + PREVENT_SERVICE, duplicated_layer_name_or_group, duplicated_layer_with_filter, - invalid_int8_primary_key, + project_invalid_pk, + project_safeguards_checks, project_trust_layer_metadata, simplify_provider_side, use_estimated_metadata, ) -from lizmap.saas import ( - SAAS_NAME, - check_project_ssl_postgis, - is_lizmap_cloud, - valid_lizmap_cloud, -) +from lizmap.saas import check_project_ssl_postgis, is_lizmap_cloud from lizmap.table_manager.base import TableManager from lizmap.table_manager.dataviz import TableManagerDataviz from lizmap.table_manager.layouts import TableManagerLayouts @@ -170,11 +172,13 @@ lizmap_user_folder, next_git_tag, qgis_version, + relative_path, to_bool, unaccent, ) from lizmap.tooltip import Tooltip from lizmap.version_checker import VersionChecker +from lizmap.widgets.check_project import Checks, Error, Severities, SourceLayer if qgis_version() >= 31400: from qgis.core import QgsProjectServerValidator @@ -202,6 +206,39 @@ def __init__(self, iface): # 04/01/2022 QgsSettings().remove('lizmap/instance_target_url_authid') + # Set some default settings when loading the plugin + beginner_mode = QgsSettings().value(Settings.key(Settings.BeginnerMode), defaultValue=None) + if beginner_mode is None: + QgsSettings().setValue(Settings.key(Settings.BeginnerMode), True) + + prevent_ecw = QgsSettings().value(Settings.key(Settings.PreventEcw), defaultValue=None) + if prevent_ecw is None: + QgsSettings().setValue(Settings.key(Settings.PreventEcw), True) + + prevent_auth_id = QgsSettings().value(Settings.key(Settings.PreventPgAuthDb), defaultValue=None) + if prevent_auth_id is None: + QgsSettings().setValue(Settings.key(Settings.PreventPgAuthDb), True) + + prevent_service = QgsSettings().value(Settings.key(Settings.PreventPgService), defaultValue=None) + if prevent_service is None: + QgsSettings().setValue(Settings.key(Settings.PreventPgService), True) + + force_pg_user_pass = QgsSettings().value(Settings.key(Settings.ForcePgUserPass), defaultValue=None) + if force_pg_user_pass is None: + QgsSettings().setValue(Settings.key(Settings.ForcePgUserPass), True) + + prevent_other_drive = QgsSettings().value(Settings.key(Settings.PreventDrive), defaultValue=None) + if prevent_other_drive is None: + QgsSettings().setValue(Settings.key(Settings.PreventDrive), True) + + allow_parent_folder = QgsSettings().value(Settings.key(Settings.AllowParentFolder), defaultValue=None) + if allow_parent_folder is None: + QgsSettings().setValue(Settings.key(Settings.AllowParentFolder), False) + + parent_folder = QgsSettings().value(Settings.key(Settings.NumberParentFolder), defaultValue=None) + if parent_folder is None: + QgsSettings().setValue(Settings.key(Settings.NumberParentFolder), 2) + # Connect the current project filepath self.current_path = None # noinspection PyUnresolvedReferences @@ -375,6 +412,7 @@ def write_log_message(message, tag, level): self.lizmap_cloud = [ self.dlg.label_lizmap_search_grant, + self.dlg.label_safe_lizmap_cloud, ] # Add widgets (not done in lizmap_var to avoid dependencies on ui) @@ -993,7 +1031,7 @@ def initGui(self): # noinspection PyUnresolvedReferences self.help_action.triggered.connect(self.show_help) - self.help_action_cloud = QAction(icon, SAAS_NAME, self.iface.mainWindow()) + self.help_action_cloud = QAction(icon, CLOUD_NAME, self.iface.mainWindow()) self.iface.pluginHelpMenu().addAction(self.help_action_cloud) # noinspection PyUnresolvedReferences self.help_action.triggered.connect(self.show_help_cloud) @@ -2895,39 +2933,17 @@ def project_config_file( if next_version != 'next': current_version = next_version - warnings = [] - log_index_panel = self.dlg.mOptionsListWidget.count() - 2 - settings_panel_name = self.dlg.mOptionsListWidget.item(self.dlg.mOptionsListWidget.count() - 1).text() - - # If the log panel must be shown to the user - # Either a warning or an error - show_log_panel = False - warning_suggest = tr('This issue not blocking the generation of the Lizmap configuration file.') - # If the user must fix some issues, the CFG will not be generated - # But with QGIS desktop < 3.22, it's not possible to do it automatically - error_cfg_saving = False - error_cfg_suggest = tr('Or use the automatic fixing button.') - error_cfg_message = tr( - "This issue must be fixed, the configuration is not going to be saved. You must visit the '{}' panel to " - "click the auto-fix button for layers currently loaded in this project." - ).format(settings_panel_name) - # Temporary disabled when used in production - qgis_min_required_not_prod_ready = 32200 if self.is_dev_version else 39900 + duplicated_in_cfg = duplicated_layer_name_or_group(self.project) + for name, count in duplicated_in_cfg.items(): + if count >= 2: + source = '"{}" → "'.format(name) + tr("count {} layers").format(count) + self.dlg.check_results.add_error(Error(source, Checks.DuplicatedLayerNameOrGroup)) # Layer ID as short name if lwc_version >= LwcVersions.Lizmap_3_6: use_layer_id, _ = self.project.readEntry('WMSUseLayerIDs', '/') if to_bool(use_layer_id, False): - show_log_panel = True - self.dlg.log_panel.append(tr('Use layer IDs as name'), Html.H2) - self.dlg.log_panel.append(warning_suggest, Html.P) - self.dlg.log_panel.append(tr( - "Since Lizmap Web Client 3.6, it's not possible anymore to use the option 'Use layer IDs " - "as name' in the project properties dialog, QGIS server tab, then WMS capabilities." - ), Html.P) - self.dlg.log_panel.append(tr( - "Please uncheck this checkbox and re-save the Lizmap configuration file."), Html.P) - warnings.append(Warnings.UseLayerIdAsName.value) + self.dlg.check_results.add_error(Error(Path(self.project.fileName()).name, Checks.WmsUseLayerIds)) target_status = self.dlg.server_combo.currentData(ServerComboData.LwcBranchStatus.value) if not target_status: @@ -2935,199 +2951,190 @@ def project_config_file( server_metadata = self.dlg.server_combo.currentData(ServerComboData.JsonMetadata.value) - if check_server and is_lizmap_cloud(server_metadata): - results, more = valid_lizmap_cloud(self.project) - if len(results): - show_log_panel = True - warnings.append(Warnings.SaasLizmapCloud.value) - self.dlg.log_panel.append(tr( - 'Some configurations are not valid with {} hosting' - ).format(SAAS_NAME), Html.H2) - self.dlg.log_panel.append(warning_suggest, Html.P) - self.dlg.log_panel.append("
") - - self.dlg.log_panel.start_table() - for i, error in enumerate(results.values()): - self.dlg.log_panel.add_row(i) - self.dlg.log_panel.append(error, Html.Td) - self.dlg.log_panel.end_row() - - self.dlg.log_panel.end_table() - self.dlg.log_panel.append("
") - - if more: - self.dlg.log_panel.append(more) - self.dlg.log_panel.append("
") - - self.dlg.log_panel.append(tr( - "The process is continuing but expect these layers to not be visible in Lizmap Web Client." - ), Html.P) + if check_server: + + # Global checks config + prevent_ecw = QgsSettings().value(Settings.key(Settings.PreventEcw), True, bool) + prevent_auth_id = QgsSettings().value(Settings.key(Settings.PreventPgAuthDb), True, bool) + prevent_service = QgsSettings().value(Settings.key(Settings.PreventPgService), True, bool) + force_pg_user_pass = QgsSettings().value(Settings.key(Settings.ForcePgUserPass), True, bool) + prevent_other_drive = QgsSettings().value(Settings.key(Settings.PreventDrive), True, bool) + allow_parent_folder = QgsSettings().value(Settings.key(Settings.AllowParentFolder), False, bool) + count_parent_folder = QgsSettings().value(Settings.key(Settings.NumberParentFolder), 2, int) + + beginner_mode = QgsSettings().value(Settings.key(Settings.BeginnerMode), True, bool) + + lizmap_cloud = is_lizmap_cloud(server_metadata) + if lizmap_cloud: + # But Lizmap Cloud override some user globals checks + prevent_ecw = True + prevent_auth_id = True + force_pg_user_pass = True + prevent_other_drive = True + if count_parent_folder > CLOUD_MAX_PARENT_FOLDER: + count_parent_folder = CLOUD_MAX_PARENT_FOLDER + # prevent_service = False We encourage service + # allow_parent_folder = False Of course we can + + summary = [] + if prevent_ecw: + summary.append(PREVENT_ECW) + if prevent_auth_id: + summary.append(PREVENT_AUTH_DB) + if prevent_service: + summary.append(PREVENT_SERVICE) + if force_pg_user_pass: + summary.append(FORCE_PG_USER_PASS) + if prevent_other_drive: + summary.append(PREVENT_OTHER_DRIVE) + if allow_parent_folder: + summary.append(ALLOW_PARENT_FOLDER + " : " + tr("{} folder(s)").format(count_parent_folder)) + else: + summary.append(FORCE_LOCAL_FOLDER) + + parent_folder = relative_path(count_parent_folder) + + results = project_safeguards_checks( + self.project, + prevent_ecw=prevent_ecw, + prevent_auth_id=prevent_auth_id, + prevent_service=prevent_service, + force_pg_user_pass=force_pg_user_pass, + prevent_other_drive=prevent_other_drive, + allow_parent_folder=allow_parent_folder, + parent_folder=parent_folder, + lizmap_cloud=lizmap_cloud, + ) + # Let's show a summary + if lizmap_cloud: + self.dlg.log_panel.append( + tr("According to global settings, overriden then by {} :").format(CLOUD_NAME), Html.P) + else: + self.dlg.log_panel.append(tr("According to global settings"), Html.P) + + self.dlg.log_panel.start_table() + for i, rule in enumerate(summary): + self.dlg.log_panel.add_row(i) + self.dlg.log_panel.append(rule, Html.Td) + self.dlg.log_panel.end_row() + self.dlg.log_panel.end_table() + + self.dlg.log_panel.append("
") + + for layer, error in results.items(): + + # Severity depends on beginner mode + severity = Severities.Blocking if beginner_mode else Severities.Important + # But override severities for Lizmap Cloud + # Because even with a 'normal' user, it won't work + override = ( + Checks.PreventEcw, Checks.PgForceUserPass, Checks.AuthenticationDb, Checks.PreventDrive) + if error in override: + severity = Severities.Blocking + + self.dlg.check_results.add_error( + Error( + layer.layer_name, + error, + source_type=SourceLayer(layer.layer_name, layer.layer_id), + ), + lizmap_cloud=lizmap_cloud, + severity=severity, + ) + + if results: + if beginner_mode: + self.dlg.log_panel.append(tr( + "The process is stopping, the CFG file is not going to be generated because some safeguards " + "are not compatible and you are using the 'Beginner' mode. Either fix these issues or switch " + "to a 'Normal' mode if you know what you are doing." + ), Html.P, level=Qgis.Critical) + else: + self.dlg.log_panel.append(tr( + "The process is continuing but these layers might be invisible if the server is not well " + "configured or the project correctly upload to the server." + ), Html.P) if check_server: error, message = check_project_ssl_postgis(self.project) - if error: - self.dlg.log_panel.append(tr('SSL connections to a PostgreSQL database'), Html.H2) - self.dlg.log_panel.append(tr( - "Connections to a PostgreSQL database hosted on {} must use a SSL secured connection." - ).format(SAAS_NAME), Html.P) - self.dlg.log_panel.append(tr( - "In the plugin, then in the '{}' panel, there is a helper to change the datasource of layers in " - "the current project only. It works only with minimum QGIS 3.22." - ).format(settings_panel_name), Html.P) - self.dlg.log_panel.append(tr( - "You must still edit your global PostgreSQL connection to allow SSL, it will take effect only " - "on newly added layer into a project." - ), Html.P) - self.dlg.log_panel.append(message, Html.P) - show_log_panel = True - - if Qgis.QGIS_VERSION_INT >= qgis_min_required_not_prod_ready: - error_cfg_saving = True - self.dlg.log_panel.append(error_cfg_suggest, Html.P) - self.dlg.log_panel.append(error_cfg_message, Html.P, level=Qgis.Critical) - self.dlg.enabled_ssl_button(True) - else: - self.dlg.log_panel.append(warning_suggest, Html.P) - - autogenerated_keys = {} - int8 = [] - for layer in self.project.mapLayers().values(): - - if not isinstance(layer, QgsVectorLayer): - continue - - result, field = auto_generated_primary_key_field(layer) - if result: - if field not in autogenerated_keys.keys(): - autogenerated_keys[field] = [] - - autogenerated_keys[field].append(layer.name()) - - if invalid_int8_primary_key(layer): - int8.append(layer.name()) - - if autogenerated_keys or int8: - show_log_panel = True - warnings.append(Warnings.InvalidFieldType.value) - self.dlg.log_panel.append(tr('Some fields are invalid for QGIS server'), Html.H2) - self.dlg.log_panel.append(warning_suggest, Html.P) - - for field, layers in autogenerated_keys.items(): - # field can be "tid", "ctid" etc - self.dlg.log_panel.append(tr( - "These layers don't have a proper primary key in the database. So QGIS Desktop tried to set a " - "temporary field called '{}' to be a unique identifier. On QGIS Server, this will bring issues." - ).format(field), Html.P) - self.dlg.log_panel.append("
") - layers.sort() - self.dlg.log_panel.start_table() - for i, layer_name in enumerate(layers): - self.dlg.log_panel.add_row(i) - self.dlg.log_panel.append(layer_name, Html.Td) - self.dlg.log_panel.end_row() - - self.dlg.log_panel.end_table() - - if int8: - int8.sort() - self.dlg.log_panel.append(tr( - "The primary key has been detected as a bigint (integer8) for your layer :"), Html.P) - self.dlg.log_panel.start_table() - for i, layer_name in enumerate(int8): - self.dlg.log_panel.add_row(i) - self.dlg.log_panel.append(layer_name, Html.Td) - self.dlg.log_panel.end_row() - self.dlg.log_panel.end_table() - self.dlg.log_panel.append("
") - - if autogenerated_keys or int8: - self.dlg.log_panel.append(tr( - "We highly recommend you to set a proper integer field as a primary key, but neither a bigint nor " - "an integer8."), Html.P) - self.dlg.log_panel.append(tr( - "The process is continuing but expect these layers to have some issues with some tools in " - "Lizmap Web Client: zoom to feature, filtering…" - ), style=Html.P) + for layer in error: + self.dlg.check_results.add_error( + Error( + layer.layer_name, + Checks.SSLConnection, + source_type=SourceLayer(layer.layer_name, layer.layer_id), + ) + ) + self.dlg.enabled_ssl_button(True) + + autogenerated_keys, int8 = project_invalid_pk(self.project) + for layer in autogenerated_keys: + self.dlg.check_results.add_error( + Error( + layer.layer_name, + Checks.MissingPk, + source_type=SourceLayer(layer.layer_name, layer.layer_id), + ) + ) + for layer in int8: + self.dlg.check_results.add_error( + Error( + layer.layer_name, + Checks.PkInt8, + source_type=SourceLayer(layer.layer_name, layer.layer_id), + ) + ) if lwc_version >= LwcVersions.Lizmap_3_7: text = duplicated_layer_with_filter(self.project) if text: self.dlg.log_panel.append(tr('Optimisation about the legend'), Html.H2) - self.dlg.log_panel.append(warning_suggest, Html.P) + (self.dlg.log_panel.append + (tr('This issue not blocking the generation of the Lizmap configuration file.'), Html.P)) self.dlg.log_panel.append(text, style=Html.P) - warnings.append(Warnings.DuplicatedLayersWithFilters.value) results = simplify_provider_side(self.project) - if len(results): - self.dlg.log_panel.append(tr('Simplify geometry on the provider side'), Html.H2) - self.dlg.log_panel.append(tr( - 'These PostgreSQL vector layers can have the simplification on the provider side') + ':', Html.P) - self.dlg.log_panel.start_table() - for i, layer in enumerate(results): - self.dlg.log_panel.add_row(i) - self.dlg.log_panel.append(layer, Html.Td) - self.dlg.log_panel.end_row() - self.dlg.log_panel.end_table() - self.dlg.log_panel.append(tr( - 'Visit the layer properties, then in the "Rendering" tab to enable it.'), Html.P) - if Qgis.QGIS_VERSION_INT >= qgis_min_required_not_prod_ready: - self.dlg.log_panel.append(error_cfg_message, Html.P, level=Qgis.Critical) - show_log_panel = True - error_cfg_saving = True + for layer in results: + self.dlg.check_results.add_error( + Error( + layer.layer_name, + Checks.SimplifyGeometry, + source_type=SourceLayer(layer.layer_name, layer.layer_id), + ) + ) self.dlg.enabled_simplify_geom(True) results = use_estimated_metadata(self.project) - if len(results): - self.dlg.log_panel.append(tr('Estimated metadata'), Html.H2) - self.dlg.log_panel.append(tr( - 'These PostgreSQL layers can have the use estimated metadata option enabled') + ':', Html.P) - self.dlg.log_panel.start_table() - for i, layer in enumerate(results): - self.dlg.log_panel.add_row(i) - self.dlg.log_panel.append(layer, Html.Td) - self.dlg.log_panel.end_row() - self.dlg.log_panel.end_table() - self.dlg.log_panel.append(tr( - 'Edit your PostgreSQL connection to enable this option, then change the datasource by right clicking ' - 'on each layer above, then click "Change datasource" in the menu. Finally reselect your layer in the ' - 'new dialog.'), Html.P) - if Qgis.QGIS_VERSION_INT >= qgis_min_required_not_prod_ready: - self.dlg.log_panel.append(error_cfg_suggest, Html.P) - self.dlg.log_panel.append(error_cfg_message, Html.P, level=Qgis.Critical) - show_log_panel = True - error_cfg_saving = True - self.dlg.enabled_estimated_md_button(True) - else: - self.dlg.log_panel.append(warning_suggest, Html.P) + for layer in results: + self.dlg.check_results.add_error( + Error( + layer.layer_name, + Checks.EstimatedMetadata, + source_type=SourceLayer(layer.layer_name, layer.layer_id), + ) + ) + self.dlg.enabled_estimated_md_button(True) - if not project_trust_layer_metadata(self.project) and Qgis.QGIS_VERSION_INT >= qgis_min_required_not_prod_ready: - self.dlg.log_panel.append(tr('Trust project metadata'), Html.H2) - self.dlg.log_panel.append(tr( - 'The project does not have the "Trust project metadata" enabled at the project level'), Html.P) - self.dlg.log_panel.append(tr( - 'In the project properties → Data sources → at the bottom, there is a checkbox to trust the project ' - 'when the layer has no metadata.'), Html.P) - self.dlg.log_panel.append(error_cfg_suggest, Html.P) - self.dlg.log_panel.append(error_cfg_message, Html.P, level=Qgis.Critical) - show_log_panel = True - error_cfg_saving = True + if not project_trust_layer_metadata(self.project): + self.dlg.check_results.add_error(Error(Path(self.project.fileName()).name, Checks.TrustProject)) self.dlg.enabled_trust_project(True) - if with_gui and show_log_panel: - self.dlg.mOptionsListWidget.setCurrentRow(log_index_panel) + self.dlg.check_results.sort() + + if with_gui and self.dlg.check_results.has_rows(): + self.dlg.mOptionsListWidget.setCurrentRow(self.dlg.mOptionsListWidget.count() - 2) + self.dlg.tab_log.setCurrentIndex(0) self.dlg.out_log.moveCursor(QTextCursor.Start) self.dlg.out_log.ensureCursorVisible() - if Qgis.QGIS_VERSION_INT >= qgis_min_required_not_prod_ready: - if with_gui and error_cfg_saving and not ignore_error: - self.dlg.log_panel.append(tr('Issues which can be fixed automatically'), Html.H2) - self.dlg.log_panel.append(tr( - 'You have issue(s) listed above, and there is a wizard to auto fix your project. Saving the ' - 'configuration file is stopping.'), Html.Strong, time=True) - self.dlg.display_message_bar( - "Error", tr('You must fix some issues about this project'), Qgis.Critical) - return None + if self.dlg.check_results.has_blocking() and not ignore_error: + self.dlg.display_message_bar( + tr("Blocking issue"), + tr("The project has at least one blocking issue. The file is not saved."), + Qgis.Critical, + ) + return None metadata = { 'qgis_desktop_version': qgis_version(), @@ -3141,14 +3148,9 @@ def project_config_file( if repository: metadata['instance_target_repository'] = repository - if valid is not None: - metadata['project_valid'] = valid - if not valid: - warnings.append(Warnings.OgcNotValid.value) - liz2json = dict() liz2json['metadata'] = metadata - liz2json['warnings'] = warnings + liz2json['warnings'] = self.dlg.check_results.to_json_summarized() liz2json["options"] = dict() liz2json["layers"] = dict() @@ -3487,26 +3489,14 @@ def check_project_validity(self): validator = QgsProjectServerValidator() valid, results = validator.validate(self.project) - if not valid: - self.dlg.log_panel.append(tr("OGC validation"), style=Html.H2) - self.dlg.log_panel.append( - tr("According to OGC standard : {}").format(tr('Valid') if valid else tr('Not valid')), Html.P) - self.dlg.log_panel.append( - tr( - "Open the 'Project properties', then 'QGIS Server' tab, at the bottom, you can check your project " - "according to OGC standard. If you need to fix a layer shortname, go to the 'Layer properties' " - "for the given layer, then 'QGIS Server' tab, edit the shortname." - ), Html.P) - LOGGER.info(f"Project has been detected : {'VALID' if valid else 'NOT valid'} according to OGC validation.") - if not valid: - message = tr( - 'The QGIS project is not valid according to OGC standards. You should check ' - 'messages in the "Project properties" → "QGIS Server" tab then "Test configuration" at the bottom. ' - '{} error(s) have been found').format(len(results)) - # noinspection PyUnresolvedReferences - self.iface.messageBar().pushMessage('Lizmap', message, level=Qgis.Warning, duration=DURATION_WARNING_BAR) + self.dlg.check_results.add_error( + Error( + Path(self.project.fileName()).name, + Checks.OgcValid, + ) + ) self.dlg.check_api_key_address() @@ -3615,6 +3605,16 @@ def save_cfg_file( Check the user defined data from GUI and save them to both global and project config files. """ + server_metadata = self.dlg.server_combo.currentData(ServerComboData.JsonMetadata.value) + self.dlg.check_results.truncate() + beginner_mode = QgsSettings().value(Settings.key(Settings.BeginnerMode), True, bool) + self.dlg.html_help.setHtml( + Checks.html( + severity=Severities.Blocking if beginner_mode else Severities.Important, + lizmap_cloud=is_lizmap_cloud(server_metadata) + ) + ) + self.dlg.log_panel.clear() self.dlg.log_panel.append(tr('Start saving the Lizmap configuration'), style=Html.P, time=True) variables = self.project.customVariables() @@ -3669,24 +3669,6 @@ def save_cfg_file( )) return False - duplicated_in_cfg = duplicated_layer_name_or_group(self.project) - message = tr('Some layer(s) or group(s) have a duplicated name in the legend.') - message += '\n\n' - message += tr( - "It's not possible to store all the Lizmap configuration for these layer(s) or group(s), you should " - "change them to make them unique and reconfigure their settings in the 'Layers' tab of the plugin.") - message += '\n\n' - display = False - for name, count in duplicated_in_cfg.items(): - if count >= 2: - display = True - message += '"{}" → "'.format(name) + tr("count {} layers").format(count) + '\n' - message += '\n\n' - message += stop_process - if display: - ScrollMessageBox(self.dlg, QMessageBox.Warning, tr('Configuration error'), message) - return False - if not self.is_dev_version: if not self.server_manager.check_lwc_version(lwc_version.value): QMessageBox.critical( diff --git a/lizmap/project_checker_tools.py b/lizmap/project_checker_tools.py index cc0df2db..e8785c2f 100644 --- a/lizmap/project_checker_tools.py +++ b/lizmap/project_checker_tools.py @@ -2,23 +2,141 @@ __license__ = 'GPL version 3' __email__ = 'info@3liz.org' -from typing import List, Optional, Tuple +from os.path import relpath +from pathlib import Path +from typing import Dict, List, Optional, Tuple from qgis.core import ( QgsDataSourceUri, QgsLayerTree, QgsMapLayer, QgsProject, + QgsProviderRegistry, + QgsRasterLayer, QgsVectorLayer, QgsWkbTypes, ) +from lizmap.definitions.lizmap_cloud import CLOUD_DOMAIN from lizmap.qgis_plugin_tools.tools.i18n import tr +from lizmap.tools import is_vector_pg, update_uri +from lizmap.widgets.check_project import Checks, SourceLayer """ Some checks which can be done on a layer. """ # https://github.com/3liz/lizmap-web-client/issues/3692 +FORCE_LOCAL_FOLDER = tr('Prevent file based layers from being in a parent folder') +ALLOW_PARENT_FOLDER = tr('Allow file based layers to be in a parent folder') +PREVENT_OTHER_DRIVE = tr('Prevent file based layers from being stored on another drive (network or local)') +PREVENT_SERVICE = tr('Prevent PostgreSQL layers from using a service file') +PREVENT_AUTH_DB = tr('Prevent PostgreSQL layers from using the QGIS authentication database') +FORCE_PG_USER_PASS = tr( + 'PostgreSQL layers, if using a user and password, must have credentials saved in the datasource') +PREVENT_ECW = tr('Prevent from using a ECW raster') + + +def project_safeguards_checks( + project: QgsProject, + prevent_ecw: bool, + prevent_auth_id: bool, + prevent_service: bool, + force_pg_user_pass: bool, + prevent_other_drive: bool, + allow_parent_folder: bool, + parent_folder: str, + lizmap_cloud: bool, +) -> Dict: + """ Check the project about safeguards. """ + # Do not use homePath, it's not designed for this if the user has set a custom home path + project_home = Path(project.absolutePath()) + results = {} + + for layer in project.mapLayers().values(): + + if isinstance(layer, QgsRasterLayer): + if layer.source().lower().endswith('ecw') and prevent_ecw: + results[SourceLayer(layer.name(), layer.id())] = Checks.PreventEcw + + if is_vector_pg(layer): + # Make a copy by using a string, so we are sure to have user or password + datasource = QgsDataSourceUri(layer.source()) + if datasource.authConfigId() != '' and prevent_auth_id: + results[SourceLayer(layer.name(), layer.id())] = Checks.AuthenticationDb + + # We can continue + continue + + if datasource.service() != '' and prevent_service: + results[SourceLayer(layer.name(), layer.id())] = Checks.PgService + + # We can continue + continue + + if datasource.host().endswith(CLOUD_DOMAIN) or force_pg_user_pass: + if not datasource.username() or not datasource.password(): + results[SourceLayer(layer.name(), layer.id())] = Checks.PgForceUserPass + + # We can continue + continue + + # Only vector/raster file based + + components = QgsProviderRegistry.instance().decodeUri(layer.dataProvider().name(), layer.source()) + if 'path' not in components.keys(): + # The layer is not file base. + continue + + layer_path = Path(components['path']) + if not layer_path.exists(): + # Let's skip, QGIS is already warning this layer + continue + + try: + relative_path = relpath(layer_path, project_home) + except ValueError: + # https://docs.python.org/3/library/os.path.html#os.path.relpath + # On Windows, ValueError is raised when path and start are on different drives. + # For instance, H: and C: + + if lizmap_cloud or prevent_other_drive: + results[SourceLayer(layer.name(), layer.id())] = Checks.PreventDrive + continue + + # Not sure what to do for now... + # We can't compute a relative path, but the user didn't enable the safety check, so we must still skip + continue + + if allow_parent_folder: + # The user allow parent folder, so we check against the string provided in the function call + if parent_folder in relative_path: + results[SourceLayer(layer.name(), layer.id())] = Checks.PreventParentFolder + else: + # The user wants only local files, we only check for ".." + if '..' in relative_path: + results[SourceLayer(layer.name(), layer.id())] = Checks.PreventParentFolder + + return results + + +def project_invalid_pk(project: QgsProject) -> Tuple[List[SourceLayer], List[SourceLayer]]: + """ Check either non existing PK or bigint. """ + autogenerated_keys = [] + int8 = [] + for layer in project.mapLayers().values(): + + if not isinstance(layer, QgsVectorLayer): + continue + + result, field = auto_generated_primary_key_field(layer) + if result: + autogenerated_keys.append(SourceLayer(layer.name(), layer.id())) + + if invalid_int8_primary_key(layer): + int8.append(SourceLayer(layer.name(), layer.id())) + + return autogenerated_keys, int8 + def auto_generated_primary_key_field(layer: QgsVectorLayer) -> Tuple[bool, Optional[str]]: """ If the primary key has been detected as tid/ctid but the field does not exist. """ @@ -148,24 +266,7 @@ def duplicated_layer_with_filter(project: QgsProject) -> Optional[str]: return text -def is_vector_pg(layer: QgsMapLayer, geometry_check=False) -> bool: - """ Return boolean if the layer is stored in PG and is a vector with a geometry. """ - if layer.type() != QgsMapLayer.VectorLayer: - return False - - if layer.dataProvider().name() != 'postgres': - return False - - if not geometry_check: - return True - - if not layer.isSpatial(): - return False - - return True - - -def simplify_provider_side(project: QgsProject, fix=False) -> List[str]: +def simplify_provider_side(project: QgsProject, fix=False) -> List[SourceLayer]: """ Return the list of layer name which can be simplified on the server side. """ results = [] for layer in project.mapLayers().values(): @@ -178,7 +279,7 @@ def simplify_provider_side(project: QgsProject, fix=False) -> List[str]: if not layer.simplifyMethod().forceLocalOptimization(): continue - results.append(layer.name()) + results.append(SourceLayer(layer.name(), layer.id())) if fix: simplify = layer.simplifyMethod() @@ -188,7 +289,7 @@ def simplify_provider_side(project: QgsProject, fix=False) -> List[str]: return results -def use_estimated_metadata(project: QgsProject, fix: bool = False) -> List[str]: +def use_estimated_metadata(project: QgsProject, fix: bool = False) -> List[SourceLayer]: """ Return the list of layer name which can use estimated metadata. """ results = [] for layer in project.mapLayers().values(): @@ -197,7 +298,7 @@ def use_estimated_metadata(project: QgsProject, fix: bool = False) -> List[str]: uri = layer.dataProvider().uri() if not uri.useEstimatedMetadata(): - results.append(layer.name()) + results.append(SourceLayer(layer.name(), layer.id())) if fix: uri.setUseEstimatedMetadata(True) @@ -213,13 +314,3 @@ def project_trust_layer_metadata(project: QgsProject, fix: bool = False) -> bool project.setTrustLayerMetadata(True) return True - - -def update_uri(layer: QgsMapLayer, uri: QgsDataSourceUri): - """ Set a new datasource URI on a layer. """ - layer.setDataSource( - uri.uri(True), - layer.name(), - layer.dataProvider().name(), - layer.dataProvider().ProviderOptions() - ) diff --git a/lizmap/resources/css/log.css b/lizmap/resources/css/log.css index 7198f5a7..4acc9465 100644 --- a/lizmap/resources/css/log.css +++ b/lizmap/resources/css/log.css @@ -75,3 +75,9 @@ hr { font-weight: bold; padding-top:25px; } + +table, th, td { + border: 1px solid black; + border-collapse: collapse; + padding:2px; +} diff --git a/lizmap/resources/ui/ui_lizmap.ui b/lizmap/resources/ui/ui_lizmap.ui index ff95796b..5d7bad21 100755 --- a/lizmap/resources/ui/ui_lizmap.ui +++ b/lizmap/resources/ui/ui_lizmap.ui @@ -245,7 +245,7 @@ QListWidget::item::selected { - 0 + 15 @@ -692,8 +692,8 @@ QListWidget::item::selected { 0 0 - 815 - 1323 + 720 + 1325 @@ -1491,8 +1491,8 @@ This is different to the map maximum extent (defined in QGIS project properties, 0 0 - 375 - 1130 + 477 + 1136 @@ -2164,8 +2164,8 @@ This is different to the map maximum extent (defined in QGIS project properties, 0 0 - 577 - 879 + 780 + 1143 @@ -3366,7 +3366,7 @@ This is different to the map maximum extent (defined in QGIS project properties, 0 0 618 - 474 + 546 @@ -3733,8 +3733,8 @@ This is different to the map maximum extent (defined in QGIS project properties, 0 0 - 302 - 991 + 386 + 867 @@ -4270,8 +4270,8 @@ This is different to the map maximum extent (defined in QGIS project properties, 0 0 - 450 - 499 + 601 + 488 @@ -4302,6 +4302,13 @@ This is different to the map maximum extent (defined in QGIS project properties, + + + + This feature is not linked to the QGIS native atlas feature. Both tools are independent. + + + @@ -4616,30 +4623,69 @@ This is different to the map maximum extent (defined in QGIS project properties, - - 0 - - - 0 - - - 0 - - - 0 - - - - - true - - - - - - Clear log + + + 0 + + + Checks + + + + + + Some checks might be blocking the CFG file generation. To fix, either use the tooltip in the last column, or check the documentation in the next tab for all errors which can be reported. + + + true + + + + + + + + + + + Help about checks + + + + + + List of checks which have been executed + + + + + + + + + + + Raw logs + + + + + + true + + + + + + + Clear log + + + + + @@ -4684,6 +4730,16 @@ This is different to the map maximum extent (defined in QGIS project properties, + + + Convert all PostgreSQL layers used in this project to use SSL + + + true + + + + @@ -4696,17 +4752,17 @@ This is different to the map maximum extent (defined in QGIS project properties, - - + + - Convert all PostgreSQL layers used in this project to use SSL + Convert all PostgreSQL layers used in this project to 'estimated metadata' true - + @@ -4719,52 +4775,180 @@ This is different to the map maximum extent (defined in QGIS project properties, - - + + - Convert all PostgreSQL layers used in this project to 'estimated metadata' + Enable the trust project option in the project properties dialog true - + Trust project - - + + - Enable the trust project option in the project properties dialog + Enable provider geometry simplification when possible in the layer properties dialog true - + Simplify geometry on the provider side - - + + + + + + + + + Safeguards + + + + + + Only if you are sure about your server and if you are more comfortable with Lizmap, you can disable some safeguards. If a project is not working, please check these safeguards. + + + true + + + + + + + SET IN PYTHON, HOSTING + + + true + + + + + + + - Enable provider geometry simplification when possible in the layer properties dialog + Beginner - - true + + + + + + Normal + + + + File based layer + + + + + + SET PYTHON PREVENT FILE IN PARENT FOLDER + + + + + + + + + Levels of parent folder allowed + + + + + + + 1 + + + 2 + + + + + + + SET PYTHON ALLOW PARENT FOLDER + + + + + + + + + + + + SET PYTHON PREVENT ANOTHER DRIVE + + + false + + + + + + + SET PYTHON PREVENT PG SERVICE FILE + + + false + + + + + + + SET PYTHON PREVENT PG AUTH DB + + + false + + + + + + + SET PYTHON USER PASSWORD DATASOURCE + + + false + + + + + + + SET PYTHON PREVENT ECW + + + @@ -4839,6 +5023,11 @@ This is different to the map maximum extent (defined in QGIS project properties,
lizmap.widgets.warning_widget
1 + + TableCheck + QTableWidget +
lizmap.widgets.check_project
+ mOptionsListWidget diff --git a/lizmap/saas.py b/lizmap/saas.py index 80d5d531..1963eaba 100644 --- a/lizmap/saas.py +++ b/lizmap/saas.py @@ -4,19 +4,14 @@ __license__ = 'GPL version 3' __email__ = 'info@3liz.org' -from os.path import relpath -from pathlib import Path -from typing import Dict, List, Tuple - -from qgis.core import ( - QgsDataSourceUri, - QgsProject, - QgsProviderRegistry, - QgsRasterLayer, -) +from typing import List, Tuple + +from qgis.core import QgsDataSourceUri, QgsProject -from lizmap.project_checker_tools import is_vector_pg, update_uri +from lizmap.definitions.lizmap_cloud import CLOUD_DOMAIN from lizmap.qgis_plugin_tools.tools.i18n import tr +from lizmap.tools import is_vector_pg, update_uri +from lizmap.widgets.check_project import SourceLayer edit_connection_title = tr("You must edit the database connection.") edit_connection = tr( @@ -29,9 +24,6 @@ "legend and click 'Change datasource' to pick the layer again with the updated connection." ) -SAAS_DOMAIN = 'lizmap.com' -SAAS_NAME = 'Lizmap Cloud' - def is_lizmap_cloud(metadata: dict) -> bool: """ Return True if the metadata is coming from Lizmap Cloud. """ @@ -39,90 +31,12 @@ def is_lizmap_cloud(metadata: dict) -> bool: # Mainly in tests? return False - return metadata.get('hosting', '') == SAAS_DOMAIN - - -def valid_lizmap_cloud(project: QgsProject) -> Tuple[Dict[str, str], str]: - """ Check the project when it's hosted on Lizmap Cloud. """ - # Do not use homePath, it's not designed for this if the user has set a custom home path - project_home = Path(project.absolutePath()) - layer_error: Dict[str, str] = {} - - connection_error = False - for layer in project.mapLayers().values(): - - if isinstance(layer, QgsRasterLayer): - if layer.source().lower().endswith('ecw'): - layer_error[layer.name()] = tr( - 'The layer "{}" is an ECW. Because of the ECW\'s licence, this format is not compatible with QGIS ' - 'server. You should switch to a COG format.').format(layer.name()) - - if is_vector_pg(layer): - datasource = QgsDataSourceUri(layer.source()) - if datasource.authConfigId() != '': - layer_error[layer.name()] = tr( - 'The layer "{}" is using the QGIS authentication database. You must either use a PostgreSQL ' - 'service or store the login and password in the layer.').format(layer.name()) - connection_error = True - - if datasource.service(): - # We trust the user about login, password etc ... - continue - - # Users might be hosted on Lizmap Cloud but using an external database - if datasource.host().endswith(SAAS_DOMAIN): - if not datasource.username() or not datasource.password(): - layer_error[layer.name()] = tr( - 'The layer "{}" is missing some credentials. Either the user and/or the password is not in ' - 'the layer datasource.' - ).format(layer.name()) - connection_error = True - - components = QgsProviderRegistry.instance().decodeUri(layer.dataProvider().name(), layer.source()) - if 'path' not in components.keys(): - # The layer is not file base. - continue - - layer_path = Path(components['path']) - if not layer_path.exists(): - # Let's skip, QGIS is already warning this layer - continue - - try: - relative_path = relpath(layer_path, project_home) - except ValueError: - # https://docs.python.org/3/library/os.path.html#os.path.relpath - # On Windows, ValueError is raised when path and start are on different drives. - # For instance, H: and C: - layer_error[layer.name()] = tr( - 'The layer "{}" can not be hosted on {} because the layer is hosted on a different drive.' - ).format(layer.name(), SAAS_NAME) - continue - - if '../../..' in relative_path: - # The layer can only be hosted the in "/qgis" directory - layer_error[layer.name()] = tr( - 'The layer "{}" can not be hosted on {} because the layer is located in too many ' - 'parent\'s folder. The current path from the project home path to the given layer is "{}".' - ).format(layer.name(), SAAS_NAME, relative_path) - - more = '' - if connection_error: - more = edit_connection_title + " " - more += edit_connection + " " - more += '
' - more += right_click_step + " " - more += tr( - "When opening a QGIS project in your desktop, you mustn't have any " - "prompt for a user&password." - ) - - return layer_error, more + return metadata.get('hosting', '') == CLOUD_DOMAIN -def check_project_ssl_postgis(project: QgsProject) -> Tuple[List[str], str]: +def check_project_ssl_postgis(project: QgsProject) -> Tuple[List[SourceLayer], str]: """ Check if the project is not using SSL on some PostGIS layers which are on a Lizmap Cloud database. """ - layer_error: List[str] = [] + layer_error: List[SourceLayer] = [] for layer in project.mapLayers().values(): if not is_vector_pg(layer): continue @@ -136,11 +50,11 @@ def check_project_ssl_postgis(project: QgsProject) -> Tuple[List[str], str]: continue # Users might be hosted on Lizmap Cloud but using an external database - if not datasource.host().endswith(SAAS_DOMAIN): + if not datasource.host().endswith(CLOUD_DOMAIN): continue if datasource.sslMode() in (QgsDataSourceUri.SslMode.SslDisable, QgsDataSourceUri.SslMode.SslAllow): - layer_error.append(layer.name()) + layer_error.append(SourceLayer(layer.name(), layer.id())) more = edit_connection_title + " " more += edit_connection + " " @@ -161,7 +75,7 @@ def fix_ssl(project: QgsProject, force: bool = True) -> int: if datasource.service(): continue - if not datasource.host().endswith(SAAS_DOMAIN): + if not datasource.host().endswith(CLOUD_DOMAIN): continue if datasource.sslMode() in (QgsDataSourceUri.SslPrefer, QgsDataSourceUri.SslMode.SslRequire): diff --git a/lizmap/test/test_table_checks.py b/lizmap/test/test_table_checks.py new file mode 100644 index 00000000..33ed41db --- /dev/null +++ b/lizmap/test/test_table_checks.py @@ -0,0 +1,60 @@ +import unittest + +from lizmap.widgets.check_project import Checks, Error, TableCheck + + +class TestProjectTable(unittest.TestCase): + + def test(self): + table = TableCheck(None) + table.setup() + + self.assertEqual(table.horizontalHeader().count(), 4) + self.assertEqual(table.verticalHeader().count(), 0) + + self.assertEqual(table.rowCount(), 0) + + table.add_error(Error('my-tailor-is-rich', Checks.DuplicatedLayerNameOrGroup)) + table.add_error(Error('home-sweet-home', Checks.DuplicatedLayerNameOrGroup)) + table.add_error(Error('home-sweet-home', Checks.MissingPk)) + self.assertEqual(table.rowCount(), 3) + + expected = [ + { + 'error': 'duplicated_layer_name_or_group', + 'level': 'project', + 'severity': 1, + 'source': 'my-tailor-is-rich', + }, { + 'severity': 1, + 'level': 'project', + 'source': 'home-sweet-home', + 'error': 'duplicated_layer_name_or_group', + }, { + 'error': 'missing_primary_key', + 'level': 'layer', + 'severity': 1, + 'source': 'home-sweet-home', + }, + ] + self.assertListEqual(expected, table.to_json()) + + expected = { + 'duplicated_layer_name_or_group': 2, + 'missing_primary_key': 1 + } + self.assertDictEqual(expected, table.to_json_summarized()) + + expected = ( + 'Validator summarized :\n\n' + '* Duplicated layer name or group → 2\n' + '* Missing a proper primary key in the database. → 1\n' + '\n') + self.assertEqual(expected, table.to_markdown_summarized()) + + +# if __name__ == '__main__': +# app = QApplication([]) +# # view = CheckProjectView() +# # view.show() +# app.exec_() diff --git a/lizmap/test/test_ui.py b/lizmap/test/test_ui.py index fd1a77b8..fa3d0fbc 100755 --- a/lizmap/test/test_ui.py +++ b/lizmap/test/test_ui.py @@ -82,7 +82,7 @@ def test_legend_options(self): self.assertIsNone(output['layers']['legend_displayed_startup'].get('noLegendImage')) # For LWC 3.5 - output = lizmap.project_config_file(LwcVersions.Lizmap_3_5, with_gui=False, check_server=False) + output = lizmap.project_config_file(LwcVersions.Lizmap_3_5, with_gui=False, check_server=False, ignore_error=True) self.assertIsNone(output['layers']['legend_displayed_startup'].get('legend_image_option')) self.assertEqual(output['layers']['legend_displayed_startup']['noLegendImage'], str(False)) diff --git a/lizmap/test/test_wizard_server.py b/lizmap/test/test_wizard_server.py index 3ac082b6..7ae1c8eb 100644 --- a/lizmap/test/test_wizard_server.py +++ b/lizmap/test/test_wizard_server.py @@ -2,7 +2,7 @@ import os import unittest -from qgis._core import Qgis +from qgis.core import Qgis from qgis.PyQt.QtCore import QUrl from qgis.PyQt.QtWidgets import QWizard diff --git a/lizmap/tools.py b/lizmap/tools.py index 874dba6a..d5f15a56 100755 --- a/lizmap/tools.py +++ b/lizmap/tools.py @@ -14,7 +14,14 @@ from pathlib import Path from typing import List, Tuple, Union -from qgis.core import Qgis, QgsApplication, QgsProviderRegistry, QgsVectorLayer +from qgis.core import ( + Qgis, + QgsApplication, + QgsDataSourceUri, + QgsMapLayer, + QgsProviderRegistry, + QgsVectorLayer, +) from qgis.PyQt.QtCore import QDateTime, QDir, Qt from lizmap.definitions.definitions import LayerProperties @@ -241,6 +248,39 @@ def merge_strings(string_1: str, string_2: str) -> str: return string_1 + (string_2 if k is None else string_2[k:]) +def relative_path(max_parent: int) -> str: + """ Return the dot notation for a maximum parent folder. """ + parent = ['..'] * max_parent + return '/'.join(parent) + + +def update_uri(layer: QgsMapLayer, uri: QgsDataSourceUri): + """ Set a new datasource URI on a layer. """ + layer.setDataSource( + uri.uri(True), + layer.name(), + layer.dataProvider().name(), + layer.dataProvider().ProviderOptions() + ) + + +def is_vector_pg(layer: QgsMapLayer, geometry_check=False) -> bool: + """ Return boolean if the layer is stored in PG and is a vector with a geometry. """ + if layer.type() != QgsMapLayer.VectorLayer: + return False + + if layer.dataProvider().name() != 'postgres': + return False + + if not geometry_check: + return True + + if not layer.isSpatial(): + return False + + return True + + def convert_lizmap_popup(content: str, layer: QgsVectorLayer) -> Tuple[str, List[str]]: """ Convert an HTML Lizmap popup to QGIS HTML Maptip. diff --git a/lizmap/widgets/check_project.py b/lizmap/widgets/check_project.py new file mode 100644 index 00000000..a04afce5 --- /dev/null +++ b/lizmap/widgets/check_project.py @@ -0,0 +1,756 @@ +__copyright__ = 'Copyright 2023, 3Liz' +__license__ = 'GPL version 3' +__email__ = 'info@3liz.org' + +from enum import Enum + +from qgis.core import ( + QgsMapLayerModel, + QgsMarkerSymbol, + QgsProject, + QgsSymbolLayerUtils, +) +from qgis.PyQt.QtCore import QSize, Qt +from qgis.PyQt.QtGui import QIcon +from qgis.PyQt.QtWidgets import ( + QAbstractItemView, + QTableWidget, + QTableWidgetItem, +) + +from lizmap.definitions.lizmap_cloud import CLOUD_MAX_PARENT_FOLDER, CLOUD_NAME +from lizmap.definitions.qgis_settings import Settings +from lizmap.qgis_plugin_tools.tools.i18n import tr +from lizmap.tools import qgis_version + + +class Header: + + """ Header in tables. """ + def __init__(self, data: str, label: str, tooltip: str): + self.data = data + self.label = label + self.tooltip = tooltip + + +class Headers(Header, Enum): + """ List of headers in the table. """ + Severity = 'severity', tr('Severity'), tr("Severity of the error") + Level = 'level', tr('Level'), tr("Level of the error") + Object = 'source', tr('Source'), tr("Source of the error") + Error = 'error', tr('Error'), tr('Description of the error') + + +class Severity: + + """ A level of severity, if it's blocking or not. """ + def __init__(self, data: int, label: str, tooltip: str, color, size: int): + self.data = data + self.label = label + self.color = color + self.size = size + self.tooltip = tooltip + + def marker(self) -> QIcon: + """ Marker used in the table. """ + pixmap = QgsSymbolLayerUtils.symbolPreviewPixmap( + QgsMarkerSymbol.createSimple( + { + "name": "circle", + "color": self.color, + "size": "{}".format(self.size), + } + ), + QSize(16, 16) + ) + return QIcon(pixmap) + + def __str__(self): + return f'Severity {self.data} : {self.label}' + + +class Severities(Severity, Enum): + """ List of severities. """ + Blocking = 0, tr('Blocking'), tr('This is blocking the CFG file'), 'red', 3 + Important = 1, tr('Important'), tr('This is important to fix, to improve performance'), 'orange', 2.5 + # Normal = 2, tr('Normal'), tr('This would be nice to have look'), 'blue', 2 + Low = 3, tr('Low'), tr('Nice to do'), 'yellow', 2 + # Some severities can only done on runtime, QGIS version and/or Lizmap Cloud + Unknown = 99, 'Unknown', 'Severity will be determined on runtime', 'green', 1 + + +class Level: + + """ Level which is raising the issue. Important to set the icon if possible. """ + def __init__(self, data: str, label: str, tooltip: str, icon: QIcon): + self.data = data + self.label = label + self.icon = icon + self.tooltip = tooltip + + def __str__(self): + return f'{self.data} : {self.label}' + + +class Levels: + + """ List of levels used. """ + + GlobalConfig = Level( + 'global', + tr('Global'), + tr('Issue in the global configuration, in QGIS or Lizmap settings'), + QIcon(':/images/themes/default/console/iconSettingsConsole.svg'), + ) + Project = Level( + 'project', + tr('Project'), + tr('Issue at the project level'), + QIcon(':/images/themes/default/mIconQgsProjectFile.svg'), + ) + Layer = Level( + 'layer', + tr('Layer'), + tr('Issue at the layer level'), + QIcon(':/images/themes/default/algorithms/mAlgorithmMergeLayers.svg'), + ) + + +class Check: + + """ Definition of a check. """ + def __init__( + self, + data: str, + title: str, + description: str, + helper: str, + level: Level, + severity: Severity, + icon: QIcon, + alt_description_lizmap_cloud: str = None, + alt_help_lizmap_cloud: str = None, + ): + self.data = data + self.title = title + self.description = description + self.alt_description = alt_description_lizmap_cloud + self.helper = helper + self.alt_help = alt_help_lizmap_cloud + self.level = level + self.severity = severity + self.icon = icon + + def description_text(self, lizmap_cloud: bool) -> str: + """ Return the best description of the check, depending on Lizmap Cloud. """ + if lizmap_cloud and self.alt_description: + return self.alt_description + else: + return self.description + + def help_text(self, lizmap_cloud: bool) -> str: + """ Return the best help of the check, depending on Lizmap Cloud. """ + if lizmap_cloud and self.alt_help: + return self.alt_help + else: + return self.helper + + def html_help(self, index: int, severity: Severity, lizmap_cloud: False) -> str: + """ HTML string to show in an HTML table. """ + row_class = '' + if index % 2: + row_class = "class=\"odd-row\"" + + html_str = ( + "" + "{title}" + "{description}" + "{how_to_fix}" + "{level}" + "{severity}" + "" + ).format( + row_class=row_class, + title=self.title, + description=self.description_text(lizmap_cloud), + how_to_fix=self.help_text(lizmap_cloud), + level=self.level.label, + severity=severity.label if self.severity == Severities.Unknown else self.severity.label, + ) + return html_str + + def html_tooltip(self, lizmap_cloud: bool = False) -> str: + """ HTML string to be used as a tooltip. """ + html_str = ( + "{description}" + "
" + "

{how_to_fix}

" + ).format( + description=self.description_text(lizmap_cloud), + how_to_fix=self.help_text(lizmap_cloud), + ) + return html_str + + def __str__(self): + return f'{self.title} : {self.description_text(False)} :{self.level} → {self.severity}' + + +# Check QGIS_VERSION_INT +qgis_32200 = tr( + 'With QGIS ≥ 3.22, you can use the auto-fix button in the "Settings" panel of the plugin to fix currently loaded ' + 'layers' +) +other_auth = tr('Either switch to another authentication mechanism') +safeguard = tr('Or disable this safeguard in your Lizmap plugin settings') +global_connection = tr( + 'To fix layers loaded later, edit your global PostgreSQL connection to enable this option, then change the ' + 'datasource by right clicking on each layer above, then click "Change datasource" in the menu. Finally reselect ' + 'your layer in the new dialog with the updated connection. When opening a QGIS project in your desktop, ' + 'you mustn\'t have any prompt for a user or password. ' + 'The edited connection will take effect only on newly added layer into a project that\'s why the right-click step ' + 'is required.' +) + + +class Checks(Check, Enum): + + """ List of checks defined. """ + + OgcValid = ( + 'ogc_validity', + tr('OGC validity'), + tr( + "According to OGC rules, the project is not valid." + ), + ( + '' + ).format( + project_properties=tr( + "Open the 'Project properties', then 'QGIS Server' tab, at the bottom, you can check your project " + "according to OGC standard" + ), + layer_shortname=tr( + "If you need to fix a layer shortname, go to the 'Layer properties' " + "for the given layer, then 'QGIS Server' tab, edit the shortname." + ), + project_shortname=tr( + "If you need to fix the project shortname, go to the 'Project properties', " + "then 'QGIS Server' tab, first tab, and change the shortname." + ), + ), + Levels.Project, + Severities.Low, + QIcon(':/images/themes/default/mIconWms.svg'), + ) + PkInt8 = ( + 'primary_key_bigint', + tr('Invalid bigint (integer8) field for QGIS Server as primary key'), + tr( + "Primary key should be an integer. If not fixed, expect layer to have some issues with some tools in " + "Lizmap Web Client: zoom to feature, filtering…" + ), + ( + '' + ).format( + help=tr( + "We highly recommend you to set a proper integer field as a primary key, but neither a bigint nor " + "an integer8." + ), + ), + Levels.Layer, + Severities.Important, + QIcon(':/images/themes/default/mIconFieldInteger.svg'), + ) + MissingPk = ( + 'missing_primary_key', + tr('Missing a proper primary key in the database.'), + tr( + "The layer must have a proper primary key defined. When it's missing, QGIS Desktop tried to set a " + "temporary field called 'tid/ctid/…' to be a unique identifier. On QGIS Server, this will bring issues." + ), + ( + '' + ).format( + help=tr( + "We highly recommend you to set a proper integer field as a primary key, but neither a bigint nor " + "an integer8." + ), + ), + Levels.Layer, + Severities.Important, + QIcon(':/images/themes/default/mSourceFields.svg'), + ) + SSLConnection = ( + 'ssl_connection', + tr('SSL connections to a PostgreSQL database'), + tr("Connections to a PostgreSQL database hosted on {} must use a SSL secured connection.").format(CLOUD_NAME), + ( + '' + ).format( + auto_fix=qgis_32200, + help=global_connection, + ), + Levels.Layer, + Severities.Blocking if qgis_version() >= 32200 else Severities.Important, + QIcon(':/images/themes/default/mIconPostgis.svg'), + ) + EstimatedMetadata = ( + 'estimated_metadata', + tr('Estimated metadata'), + tr("PostgreSQL layer can have the use estimated metadata option enabled"), + ( + '' + ).format( + auto_fix=qgis_32200, + help=global_connection, + ), + Levels.Layer, + Severities.Blocking if qgis_version() >= 32200 else Severities.Important, + QIcon(':/images/themes/default/mIconPostgis.svg'), + ) + SimplifyGeometry = ( + 'simplify_geometry', + tr('Simplify geometry on the provider side'), + tr("PostgreSQL layer can have the simplification on the provider side enabled"), + ( + '' + ).format( + auto_fix=qgis_32200, + help=tr( + 'Visit the layer properties, then in the "Rendering" tab to enable it simplification on the provider ' + 'side on the given layer.' + ), + ), + Levels.Layer, + Severities.Blocking if qgis_version() >= 32200 else Severities.Important, + QIcon(':/images/themes/default/mIconGeometryCollectionLayer.svg'), + ) + DuplicatedLayerNameOrGroup = ( + 'duplicated_layer_name_or_group', + tr('Duplicated layer name or group'), + tr("It's not possible to store all the Lizmap configuration for these layer(s) or group(s)."), + ( + '' + ).format( + tr('You must change them to make them unique'), + tr('Reconfigure their settings in the "Layers" tab of the plugin') + ), + Levels.Project, + Severities.Important, + QIcon(':/images/themes/default/propertyicons/editmetadata.svg'), + ) + WmsUseLayerIds = ( + 'wms_use_layer_id', + tr('Do not use layer IDs as name'), + tr( + "It's not possible anymore to use the option 'Use layer IDs as name' in the project properties dialog, " + "QGIS server tab, then WMS capabilities." + ), + ''.format( + help=tr("Uncheck this checkbox and re-save the Lizmap configuration file") + ), + Levels.Project, + Severities.Blocking, + QIcon(':/images/themes/default/mIconWms.svg'), + ) + TrustProject = ( + 'trust_project_metadata', + tr('Trust project metadata'), + tr('The project does not have the "Trust project metadata" enabled at the project level'), + ( + ''.format( + help=tr( + 'In the project properties → Data sources → at the bottom, there is a checkbox to trust the ' + 'project when the layer has no metadata.' + ), + auto_fix=qgis_32200, + ) + ), + Levels.Project, + Severities.Blocking if qgis_version() >= 32200 else Severities.Important, + QIcon(':/images/themes/default/mIconQgsProjectFile.svg'), + ) + PreventEcw = ( + Settings.PreventEcw, + tr('Prevent ECW raster'), + tr( + 'The layer is using the ECW raster format. Because of the ECW\'s licence, this format is not compatible ' + 'with most of QGIS server installations. You have activated a safeguard about preventing you using an ' + 'ECW layer.'), + ( + ''.format( + help=tr('Either switch to a COG format'), + other=safeguard, + ) + ), + Levels.Layer, + Severities.Unknown, + QIcon(':/images/themes/default/mIconRasterLayer.svg'), + tr( + 'The layer is using an ECW raster format. Because of the ECW\'s licence, this format is not compatible ' + 'with QGIS server.' + ), + ( + '' + ).format(help=tr('Switch to a COG format')) + ) + AuthenticationDb = ( + Settings.PreventPgAuthDb, + tr('Authentication database'), + tr( + 'The layer is using the QGIS authentication database. You have activated a safeguard preventing you using ' + 'the QGIS authentication database.' + ), + ( + ''.format( + help=other_auth, + other=safeguard, + global_connection=global_connection, + ) + ), + Levels.Layer, + Severities.Unknown, + QIcon(':/images/themes/default/mIconPostgis.svg'), + tr('The layer is using the QGIS authentication database. This is not compatible with {}').format(CLOUD_NAME), + ( + '' + ).format( + service=tr('Either use a PostgreSQL service'), + login_pass=tr('Or store the login and password in the layer.') + ) + ) + PgService = ( + Settings.PreventPgService, + tr('PostgreSQL service'), + tr( + 'Using a PostgreSQL service file can be recommended in many cases, but it requires a configuration step. ' + 'If you have done the configuration (on the server side mainly), you can disable this safeguard.' + ), + ( + ''.format( + help=other_auth, + other=safeguard, + global_connection=global_connection, + ) + ), + Levels.Layer, + Severities.Unknown, + QIcon(':/images/themes/default/mIconPostgis.svg'), + ) + PgForceUserPass = ( + Settings.ForcePgUserPass, + tr('PostgreSQL user and/or password'), + tr( + 'The layer is missing some credentials, either user and/or password.' + ), + ( + ''.format( + edit_layer=tr('Edit your layer configuration by force saving user&password'), + help=other_auth, + other=safeguard, + global_connection=global_connection, + ) + ), + Levels.Layer, + Severities.Unknown, + QIcon(':/images/themes/default/mIconPostgis.svg'), + ) + PreventDrive = ( + Settings.PreventDrive, + tr('Other drive (network or local)'), + tr('The layer is stored on another drive.'), + ( + ''.format( + help=tr('Either move this file based layer'), + other=safeguard, + ) + ), + Levels.Layer, + Severities.Unknown, + QIcon(':/qt-project.org/styles/commonstyle/images/networkdrive-16.png'), + tr('The layer is stored on another drive, which is not possible using {}.').format(CLOUD_NAME), + ( + '' + ).format( + help=tr('Move the layer'), + ) + ) + PreventParentFolder = ( + Settings.AllowParentFolder, + tr('Parent folder'), + tr('The layer is stored in too many parent\'s folder, compare to the QGS file.'), + ( + ''.format( + help=tr('Either move this file based layer'), + other=safeguard, + ) + ), + Levels.Layer, + Severities.Unknown, + QIcon(':/images/themes/default/mIconFolderOpen.svg'), + tr('The layer is stored in too many parent\'s folder, compare to the QGS file.'), + ( + '' + ).format( + help=tr('Either move the layer'), + other=safeguard, + fyi=tr( + 'For your information, the maximum of parents is {count} on {hosting_name}. This will be overriden ' + 'on runtime if you use a higher value according to the server selected in the first panel.' + ).format( + count=CLOUD_MAX_PARENT_FOLDER, + hosting_name=CLOUD_NAME + ), + ) + ) + + @classmethod + def html(cls, severity: Severity, lizmap_cloud: bool) -> str: + """ Generate an HTML table, according to the instance. """ + html_str = '' + html_str += ( + '' + ).format( + title=tr('Title'), + description=tr('Description'), + howto=tr('How to fix'), + level=tr('Level'), + severity=tr('Severity'), + ) + copy_sort = list(cls.__members__.values()) + copy_sort.sort(key=lambda x: severity.data if x.severity == Severities.Unknown else x.severity.data) + for i, check in enumerate(copy_sort): + html_str += check.html_help(i, severity, lizmap_cloud) + html_str += '
{title}{description}{howto}{level}{severity}
' + return html_str + + +class SourceLayer: + + """ For identifying a layer in a project. """ + def __init__(self, layer_name, layer_id): + self.layer_id = layer_id + self.layer_name = layer_name + + +class SourceType: + + """ List of sources in the project. """ + + Layer = SourceLayer + + +class Error: + + """ An error is defined by a check and a source. """ + def __init__(self, source: str, check: Check, source_type=None): + self.source = source + self.check = check + self.source_type = source_type + + def __str__(self): + return f'{self.source} : {self.check}' + + +class TableCheck(QTableWidget): + + """ Subclassing of QTableWidget in the plugin. """ + + # noinspection PyUnresolvedReferences + DATA = Qt.UserRole + JSON = DATA + 1 + + def setup(self): + """ Setting up parameters. """ + # Do not use the constructor __init__, it's not working. Maybe because of UI files ? + + self.setSelectionMode(QAbstractItemView.SingleSelection) + self.setEditTriggers(QAbstractItemView.NoEditTriggers) + self.setSelectionBehavior(QAbstractItemView.SelectRows) + self.setAlternatingRowColors(True) + self.horizontalHeader().setStretchLastSection(True) + self.horizontalHeader().setVisible(True) + # Bug, same as self.sort() + # self.setSortingEnabled(True) + + self.setColumnCount(len(Headers)) + for i, header in enumerate(Headers): + column = QTableWidgetItem(header.label) + column.setToolTip(header.tooltip) + self.setHorizontalHeaderItem(i, column) + + def truncate(self): + """ Truncate the table. """ + self.setRowCount(0) + + def has_blocking(self) -> bool: + """ If the table has at least one blocking issue. """ + for row in range(self.rowCount()): + if self.item(row, 0).data(self.DATA) == 0: + return True + return False + + def has_rows(self) -> int: + """ If the table has at least one row displayed. """ + return self.rowCount() + + def sort(self): + """ Sort the table by severity. """ + # Strange bug occurring when we launch the analysis on the second time + # Lines are disappearing + # self.sortByColumn(0, Qt.AscendingOrder) + pass + + def to_json(self) -> list: + """ Export data to JSON. """ + result = [] + + for row in range(self.rowCount()): + data = dict() + for i, header in enumerate(Headers): + data[header.data] = self.item(row, i).data(self.JSON) + result.append(data) + + return result + + def to_json_summarized(self) -> dict: + """ Export a sum up of warnings to JSON. """ + result = {} + for row in range(self.rowCount()): + error_id = self.item(row, 3).data(self.JSON) + if error_id not in result.keys(): + result[error_id] = 1 + else: + result[error_id] += 1 + return result + + def to_markdown_summarized(self) -> str: + """ Export a sum up of warnings to JSON. """ + result = {} + for row in range(self.rowCount()): + error_name = self.item(row, 3).data(Qt.DisplayRole) + if error_name not in result.keys(): + result[error_name] = 1 + else: + result[error_name] += 1 + + text = 'Validator summarized :\n\n' + for error_name, count in result.items(): + text += '* {} → {}\n'.format(error_name, count) + text += '\n' + return text + + def add_error(self, error: Error, lizmap_cloud: bool = False, severity=None): + """ Add an error in the table. """ + # By default, let's take the one in the error + used_severity = error.check.severity + if used_severity == Severities.Unknown: + if severity: + # The given severity is overriden the one in the error + used_severity = severity + else: + raise NotImplementedError('Missing severity level') + + row = self.rowCount() + self.setRowCount(row + 1) + + column = 0 + + # Severity + item = QTableWidgetItem(used_severity.label) + item.setData(self.DATA, used_severity.data) + item.setData(self.JSON, used_severity.data) + item.setToolTip(used_severity.tooltip) + item.setIcon(used_severity.marker()) + self.setItem(row, column, item) + column += 1 + + # Level + item = QTableWidgetItem(error.check.level.label) + item.setData(self.DATA, error.check.level.data) + item.setData(self.JSON, error.check.level.data) + item.setToolTip(error.check.level.tooltip) + item.setIcon(error.check.level.icon) + self.setItem(row, column, item) + column += 1 + + # Source + item = QTableWidgetItem(error.source) + item.setData(self.DATA, error.source) + if isinstance(error.source_type, SourceType.Layer): + item.setToolTip(error.source_type.layer_id) + layer = QgsProject.instance().mapLayer(error.source_type.layer_id) + item.setIcon(QgsMapLayerModel.iconForLayer(layer)) + item.setData(self.JSON, error.source_type.layer_id) + else: + # Project only for now + # TODO fix else + item.setData(self.JSON, error.source) + self.setItem(row, column, item) + column += 1 + + # Error + item = QTableWidgetItem(error.check.title) + item.setData(self.DATA, error.source) + item.setData(self.JSON, error.check.data) + item.setToolTip(error.check.html_tooltip(lizmap_cloud)) + if error.check.icon: + item.setIcon(error.check.icon) + self.setItem(row, column, item) + column += 1