From 45e2da49ddb1c4b0f047449ac092a6b397afd28b Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 11 Dec 2024 14:13:35 +0100 Subject: [PATCH 001/245] add new fonctions for selection export --- qgepqwat2ili/utils/qgep_export_utils.py | 378 ++++++++++++++++++++++++ 1 file changed, 378 insertions(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 5c0eec9a..af0cda21 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -125,6 +125,384 @@ def check_fk_in_subsetid(self, subset, relation): else: return self.tid_maker.tid_for_row(relation) +# 10.12.2024 +def get_selection_text_for_in_statement(selection_list): + """ + convert selection_list to selection_text to fit SQL IN statement + """ + selection_text = "" + + for list_item in selection_list: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] + + logger.debug(f"selection_text = {selection_text} ...") + return selection_text + + +# 12.11.2024 to clean up - get_ws_wn_ids kann das auch +def get_cl_re_ids(classname): + """ + Get list of id's of reaches of the channels provided + """ + + # define classes that this is allowed to use - adapt for TWW to include model changes + if classname == "channel": + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + cl_re_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # cl_re_ids_count = int(cursor.fetchone()[0]) + # if cl_re_ids_count == 0: + if cursor.fetchone() is None: + cl_re_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + cl_re_ids.append(strrow) + logger.debug(f" building up '{cl_re_ids}' ...") + + return cl_re_ids + else: + logger.warning(f"Do not use this function with {classname} !") + return None + + +# 10.12.2024 +def get_connected_we_from_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_from_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected from wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_from_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_from_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + + return connected_wn_from_re_ids + + +# 10.12.2024 +def get_connected_overflow_to_wn_ids(selected_ids): + """ + Get all connected wastewater_nodes from overflows.fk_overflow_to + """ + + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_overflow_to_wn_ids = [] + + subset_text = get_selection_text_for_in_statement(selected_ids) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT ov.fk_overflow_to FROM tww_od.wastewater_node wn LEFT JOIN tww_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ov.fk_overflow_to FROM tww_od.wastewater_node wn LEFT JOIN tww_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + + return connected_overflow_to_wn_ids + + +# 10.12.2024 +def get_connected_we_to_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_to_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + + return connected_wn_to_re_ids + + +def get_ws_wn_ids(classname): + """ + Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) + """ + + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + ws_wn_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_wn_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_wn_ids + + +def get_ws_selected_ww_networkelements(selected_wwn): + """ + Get list of id's of wastewater_structure from selected wastewater_network_elements + """ + + logger.debug( + f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + selection_text = "" + + for list_item in selected_wwn: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] + + logger.debug(f"selection_text = {selection_text} ...") + + ws_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_ids + + +# 10.1.2024 +def filter_reaches(selected_ids): + """ + Filter out reaches from selected_ids + """ + + logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") + + if selected_ids is None: + subset_reaches_ids = None + else: + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + subset_reaches_ids = [] + + get_selection_text_for_in_statement(selected_ids) + + # select all reaches + cursor.execute(f"SELECT obj_id FROM qgep_od.reach;") + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + all_reaches_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute(f"SELECT obj_id FROM qgep_od.reach;") + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + all_reaches_ids.append(strrow) + logger.debug(f" building up '{all_reaches_ids}' ...") + + for list_item in selected_ids: + if list_item in all_reaches_ids: + subset_reaches_ids.append(list_item) + logger.debug( + f"'filter_reaches: {list_item}' is a reach id - added to subset_reaches_ids" + ) + else: + logger.debug(f"'filter_reaches: {list_item}' is not a reach id") + + return subset_reaches_ids + + +def remove_from_selection(selected_ids, remove_ids): + """ + Remove ids from selected_ids if they are in selected_ids + """ + + for list_item in remove_ids: + # selected_ids = selected_ids.remove(list_item) + try: + selected_ids.remove(list_item) + except Exception: + logger.debug( + f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" + ) + + return selected_ids + + +def add_to_selection(selected_ids, add_ids): + """ + Append ids to selected_ids + """ + + if selected_ids is None: + selected_ids = [] + + for list_item in add_ids: + # selected_ids = selected_ids.append(list_item) + selected_ids.append(list_item) + + return selected_ids + + def create_metaattributes(self, row): metaattribute = self.abwasser_model.metaattribute( # FIELDS TO MAP TO ABWASSER.metaattribute From a2efc3e88edfd1788d355a913a3444a1e4fe6177 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 11 Dec 2024 14:19:19 +0100 Subject: [PATCH 002/245] add fonctions and flag_approach_urs --- qgepqwat2ili/qgepsia405/export.py | 112 ++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index e1e93382..201e7bf5 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -7,6 +7,20 @@ from .. import utils from ..utils.basket_utils import BasketUtils from ..utils.ili2db import skip_wwtp_structure_ids + +# 4.10.2024 +# 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added +from ..utils.ili2db import ( + add_to_selection, + filter_reaches, + get_connected_overflow_to_wn_ids, + get_connected_we_from_re, + get_connected_we_to_re, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, +) + from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model @@ -42,18 +56,102 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske current_basket = basket_utils.basket_topic_sia405_abwasser - # Filtering + # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] - # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure + flag_approach_urs = True + if flag_approach_urs: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) +# https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # 7. If extra_reaches then remove from adapted_subset_ids + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.debug( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") - wastewater_structure_id_sia405abwasser_list = None - wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() + # 10. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) + else: # flag_approach_urs = False + # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + + # 3. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + + # 4. check if filtered + if filtered: + if ws_off_sia405abwasser_list: + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + # else do nothing + else: + if ws_off_sia405abwasser_list: + # add all data of wastewater_structures to selection + subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) + logger.debug( + f"subset_ids of all wws : {subset_ids}", + ) + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + ) + # add reach_ids + # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) + # treat export as with a selection + filtered = True + + # else do nothing + + # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) + subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) + logger.debug( + f"subset_wws_ids: {subset_wws_ids}", + ) + subset_ids = add_to_selection(subset_ids, subset_wws_ids) + logger.debug( + f"subset_ids with wws : {subset_ids}", + ) - logger.info( - f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", - ) # Orientation oriented = orientation is not None From 08d6065cdd37cb87273be60c77e04f06980f6ee5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:20:14 +0000 Subject: [PATCH 003/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 7 ++----- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 201e7bf5..2d63c413 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -6,7 +6,6 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids # 4.10.2024 # 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added @@ -20,7 +19,6 @@ get_ws_wn_ids, remove_from_selection, ) - from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model @@ -75,7 +73,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) -# https://www.w3schools.com/python/ref_set_difference.asp + # https://www.w3schools.com/python/ref_set_difference.asp # x = {"apple", "banana", "cherry"} # y = {"google", "microsoft", "apple"} # z = x.difference(y) @@ -107,7 +105,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.debug( f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", ) - else: # flag_approach_urs = False + else: # flag_approach_urs = False # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) ws_off_sia405abwasser_list = None ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") @@ -152,7 +150,6 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske f"subset_ids with wws : {subset_ids}", ) - # Orientation oriented = orientation is not None if oriented: diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index af0cda21..bbf1942e 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -125,6 +125,7 @@ def check_fk_in_subsetid(self, subset, relation): else: return self.tid_maker.tid_for_row(relation) + # 10.12.2024 def get_selection_text_for_in_statement(selection_list): """ @@ -502,7 +503,6 @@ def add_to_selection(selected_ids, add_ids): return selected_ids - def create_metaattributes(self, row): metaattribute = self.abwasser_model.metaattribute( # FIELDS TO MAP TO ABWASSER.metaattribute From f9aafcec10b5f0753c0bc36bbf424ca906f161bd Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 11 Dec 2024 14:21:59 +0100 Subject: [PATCH 004/245] import psycopg2 --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index af0cda21..e511092b 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1,5 +1,5 @@ import json - +import psycopg2 from geoalchemy2.functions import ST_Force2D, ST_GeomFromGeoJSON from sqlalchemy import or_ From aaf5354fa2f8c7b5c7dbfa5139706bcb85d5094d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:25:00 +0000 Subject: [PATCH 005/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 56f4b3b4..8941b9f0 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1,4 +1,5 @@ import json + import psycopg2 from geoalchemy2.functions import ST_Force2D, ST_GeomFromGeoJSON from sqlalchemy import or_ From a8cb101948d7ea4f6cb25fba2350d735884bd869 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 11 Dec 2024 14:28:47 +0100 Subject: [PATCH 006/245] import from .various get_pgconf_as_psycopg2_dsn --- qgepqwat2ili/utils/qgep_export_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 56f4b3b4..73158237 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -3,8 +3,7 @@ from geoalchemy2.functions import ST_Force2D, ST_GeomFromGeoJSON from sqlalchemy import or_ -from .various import logger - +from .various import logger, get_pgconf_as_psycopg2_dsn class QgepExportUtils: @@ -438,7 +437,7 @@ def filter_reaches(selected_ids): get_selection_text_for_in_statement(selected_ids) # select all reaches - cursor.execute(f"SELECT obj_id FROM qgep_od.reach;") + cursor.execute("SELECT obj_id FROM qgep_od.reach;") # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ # ws_wn_ids_count = int(cursor.fetchone()[0]) @@ -448,7 +447,7 @@ def filter_reaches(selected_ids): else: # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute(f"SELECT obj_id FROM qgep_od.reach;") + cursor.execute("SELECT obj_id FROM qgep_od.reach;") records = cursor.fetchall() # 15.11.2024 - does not get all records, but only n-1 From bc1bb017f38c6e648e049c44962f9d4aaab01d20 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:29:58 +0000 Subject: [PATCH 007/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 0fea07af..cb59258d 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -4,7 +4,8 @@ from geoalchemy2.functions import ST_Force2D, ST_GeomFromGeoJSON from sqlalchemy import or_ -from .various import logger, get_pgconf_as_psycopg2_dsn +from .various import get_pgconf_as_psycopg2_dsn, logger + class QgepExportUtils: From 7408db5553ce5b421e0203e071574dd351ab609f Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 09:26:16 +0100 Subject: [PATCH 008/245] adapted references --- qgepqwat2ili/qgepsia405/export.py | 15 +++++++++++++-- qgepqwat2ili/utils/qgep_export_utils.py | 9 +++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 2d63c413..33b2179f 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -9,7 +9,19 @@ # 4.10.2024 # 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added -from ..utils.ili2db import ( +# from ..utils.ili2db import ( + # add_to_selection, + # filter_reaches, + # get_connected_overflow_to_wn_ids, + # get_connected_we_from_re, + # get_connected_we_to_re, + # get_ws_selected_ww_networkelements, + # get_ws_wn_ids, + # remove_from_selection, +# ) +from ..utils.qgep_export_utils import QgepExportUtils +# 12.12.2024 - verschoben +from ..utils.qgep_export_utils import ( add_to_selection, filter_reaches, get_connected_overflow_to_wn_ids, @@ -19,7 +31,6 @@ get_ws_wn_ids, remove_from_selection, ) -from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index cb59258d..886d80cb 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -179,8 +179,9 @@ def get_cl_re_ids(classname): # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) cl_re_ids.append(strrow) - logger.debug(f" building up '{cl_re_ids}' ...") + # logger.debug(f" building up '{cl_re_ids}' ...") + logger.info(f" cl_re_ids: '{cl_re_ids}'") return cl_re_ids else: logger.warning(f"Do not use this function with {classname} !") @@ -230,7 +231,7 @@ def get_connected_we_from_re(subset_reaches): if strrow is not None: connected_wn_from_re_ids.append(strrow) logger.debug(f" building up '{connected_wn_from_re_ids}' ...") - + logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") return connected_wn_from_re_ids @@ -277,7 +278,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): if strrow is not None: connected_overflow_to_wn_ids.append(strrow) logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") - + logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") return connected_overflow_to_wn_ids @@ -324,7 +325,7 @@ def get_connected_we_to_re(subset_reaches): if strrow is not None: connected_wn_to_re_ids.append(strrow) logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - + logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids From 778b9a4240dd9193634e1685f8871fb1641929b2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 08:26:40 +0000 Subject: [PATCH 009/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 33b2179f..28701ccd 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -7,21 +7,10 @@ from .. import utils from ..utils.basket_utils import BasketUtils -# 4.10.2024 -# 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added -# from ..utils.ili2db import ( - # add_to_selection, - # filter_reaches, - # get_connected_overflow_to_wn_ids, - # get_connected_we_from_re, - # get_connected_we_to_re, - # get_ws_selected_ww_networkelements, - # get_ws_wn_ids, - # remove_from_selection, -# ) -from ..utils.qgep_export_utils import QgepExportUtils # 12.12.2024 - verschoben +# ) from ..utils.qgep_export_utils import ( + QgepExportUtils, add_to_selection, filter_reaches, get_connected_overflow_to_wn_ids, @@ -35,6 +24,18 @@ from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model +# 4.10.2024 +# 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added +# from ..utils.ili2db import ( +# add_to_selection, +# filter_reaches, +# get_connected_overflow_to_wn_ids, +# get_connected_we_from_re, +# get_connected_we_to_re, +# get_ws_selected_ww_networkelements, +# get_ws_wn_ids, +# remove_from_selection, + def qgep_export_sia405(selection=None, labels_file=None, orientation=None, basket_enabled=False): """ From 37868dd404d1c71998e958c4c0364f3946f96fc3 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 09:30:31 +0100 Subject: [PATCH 010/245] correct queries qgep_od instead of tww_od --- qgepqwat2ili/utils/qgep_export_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 886d80cb..3b757178 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -254,7 +254,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( - f"SELECT ov.fk_overflow_to FROM tww_od.wastewater_node wn LEFT JOIN tww_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ @@ -266,7 +266,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working cursor.execute( - f"SELECT ov.fk_overflow_to FROM tww_od.wastewater_node wn LEFT JOIN tww_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" ) records = cursor.fetchall() From 520ee117ab8776d79039ff9db2086b60e71f6273 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 09:40:36 +0100 Subject: [PATCH 011/245] initialized all_reaches_ids = [] - added logger.info --- qgepqwat2ili/utils/qgep_export_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 3b757178..6eb9183f 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -436,7 +436,8 @@ def filter_reaches(selected_ids): cursor = connection.cursor() subset_reaches_ids = [] - + all_reaches_ids = [] + get_selection_text_for_in_statement(selected_ids) # select all reaches @@ -470,7 +471,7 @@ def filter_reaches(selected_ids): ) else: logger.debug(f"'filter_reaches: {list_item}' is not a reach id") - + logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") return subset_reaches_ids From 1d82a9b1102a0d42a0488ec77fab2752169cf8cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 08:44:43 +0000 Subject: [PATCH 012/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 6eb9183f..35ef0316 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -437,7 +437,7 @@ def filter_reaches(selected_ids): subset_reaches_ids = [] all_reaches_ids = [] - + get_selection_text_for_in_statement(selected_ids) # select all reaches From 967e3205b9a99ab9dd681885523f7225aa868e5f Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 09:58:46 +0100 Subject: [PATCH 013/245] difference method needs sets instead of lists - converted lists to sets --- qgepqwat2ili/qgepsia405/export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 28701ccd..48b05814 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -85,11 +85,16 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets # https://www.w3schools.com/python/ref_set_difference.asp # x = {"apple", "banana", "cherry"} # y = {"google", "microsoft", "apple"} # z = x.difference(y) extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) # 7. If extra_reaches then remove from adapted_subset_ids if not extra_reaches_ids: # list is empty - no need for adaption @@ -97,11 +102,14 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske "no extra reaches - so nothing to remove from adapted_subset_ids", ) else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) # if len(extra_reaches_ids) > 0: adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) - logger.debug( + logger.info( f"subset_wws_ids: {subset_wws_ids}", ) # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) @@ -109,12 +117,12 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") # 10. Show ws_off_sia405abwasser_list - logger.debug( + logger.info( f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) - logger.debug( + logger.info( f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", ) else: # flag_approach_urs = False From ce0f6ddec16a11786669b3abe16ffdd0d469045a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 08:59:12 +0000 Subject: [PATCH 014/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 48b05814..a39c70e4 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -104,7 +104,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske else: logger.debug( f"extra_reaches_ids: {extra_reaches_ids} found!", - ) + ) # if len(extra_reaches_ids) > 0: adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) # 8. get all id's of connected wastewater_structures From ca6e2c56b84c6be2708c486468d4134c032faa3a Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:03:17 +0100 Subject: [PATCH 015/245] replaced with code that first converts to sets --- qgepqwat2ili/qgepsia405/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 48b05814..60864ec5 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -91,7 +91,8 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # x = {"apple", "banana", "cherry"} # y = {"google", "microsoft", "apple"} # z = x.difference(y) - extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # replaced with code that first converts to sets + #extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) From a1229048a036aedd21f546b79eec35002d2e5e7b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:03:42 +0000 Subject: [PATCH 016/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 73b63256..be329fd9 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -92,7 +92,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # y = {"google", "microsoft", "apple"} # z = x.difference(y) # replaced with code that first converts to sets - #extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) From f8d6de94f6c47b99f093f98a6e75a55171a4aa02 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:21:03 +0100 Subject: [PATCH 017/245] check for None ; if remove_ids is None: --- qgepqwat2ili/utils/qgep_export_utils.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 35ef0316..771434a1 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -480,16 +480,19 @@ def remove_from_selection(selected_ids, remove_ids): Remove ids from selected_ids if they are in selected_ids """ - for list_item in remove_ids: - # selected_ids = selected_ids.remove(list_item) - try: - selected_ids.remove(list_item) - except Exception: - logger.debug( - f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" - ) + if remove_ids is None: + return selected_ids + else: + for list_item in remove_ids: + # selected_ids = selected_ids.remove(list_item) + try: + selected_ids.remove(list_item) + except Exception: + logger.debug( + f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" + ) - return selected_ids + return selected_ids def add_to_selection(selected_ids, add_ids): From 84b8bc501302ad8cbe49514a73f33b96f470f73b Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:25:17 +0100 Subject: [PATCH 018/245] intend new fonctions - to be also part of class QgepExportUtils --- qgepqwat2ili/utils/qgep_export_utils.py | 694 ++++++++++++------------ 1 file changed, 347 insertions(+), 347 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 771434a1..a1cdf6f7 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -127,331 +127,236 @@ def check_fk_in_subsetid(self, subset, relation): return self.tid_maker.tid_for_row(relation) -# 10.12.2024 -def get_selection_text_for_in_statement(selection_list): - """ - convert selection_list to selection_text to fit SQL IN statement - """ - selection_text = "" + # 10.12.2024 + def get_selection_text_for_in_statement(selection_list): + """ + convert selection_list to selection_text to fit SQL IN statement + """ + selection_text = "" + + for list_item in selection_list: + selection_text += "'" + selection_text += list_item + selection_text += "'," - for list_item in selection_list: - selection_text += "'" - selection_text += list_item - selection_text += "'," + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] + logger.debug(f"selection_text = {selection_text} ...") + return selection_text - logger.debug(f"selection_text = {selection_text} ...") - return selection_text + # 12.11.2024 to clean up - get_ws_wn_ids kann das auch + def get_cl_re_ids(classname): + """ + Get list of id's of reaches of the channels provided + """ -# 12.11.2024 to clean up - get_ws_wn_ids kann das auch -def get_cl_re_ids(classname): - """ - Get list of id's of reaches of the channels provided - """ + # define classes that this is allowed to use - adapt for TWW to include model changes + if classname == "channel": + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") - # define classes that this is allowed to use - adapt for TWW to include model changes - if classname == "channel": - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + cl_re_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # cl_re_ids_count = int(cursor.fetchone()[0]) + # if cl_re_ids_count == 0: + if cursor.fetchone() is None: + cl_re_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + cl_re_ids.append(strrow) + # logger.debug(f" building up '{cl_re_ids}' ...") + + logger.info(f" cl_re_ids: '{cl_re_ids}'") + return cl_re_ids + else: + logger.warning(f"Do not use this function with {classname} !") + return None + + + # 10.12.2024 + def get_connected_we_from_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() - cl_re_ids = [] + connected_wn_from_re_ids = [] - # select all obj_id of the wastewater_nodes of wwtp_structure + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected from wastewater_nodes from provided subset of reaches cursor.execute( - "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # cl_re_ids_count = int(cursor.fetchone()[0]) - # if cl_re_ids_count == 0: + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: if cursor.fetchone() is None: - cl_re_ids = None + connected_wn_from_re_ids = None else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 for row in records: logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) - cl_re_ids.append(strrow) - # logger.debug(f" building up '{cl_re_ids}' ...") - - logger.info(f" cl_re_ids: '{cl_re_ids}'") - return cl_re_ids - else: - logger.warning(f"Do not use this function with {classname} !") - return None - - -# 10.12.2024 -def get_connected_we_from_re(subset_reaches): - """ - Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches - """ - - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_from_re_ids = [] - - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - - # select all connected from wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_from_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + if strrow is not None: + connected_wn_from_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") + return connected_wn_from_re_ids + + + # 10.12.2024 + def get_connected_overflow_to_wn_ids(selected_ids): + """ + Get all connected wastewater_nodes from overflows.fk_overflow_to + """ + + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_from_re_ids.append(strrow) - logger.debug(f" building up '{connected_wn_from_re_ids}' ...") - logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") - return connected_wn_from_re_ids - - -# 10.12.2024 -def get_connected_overflow_to_wn_ids(selected_ids): - """ - Get all connected wastewater_nodes from overflows.fk_overflow_to - """ - - logger.info( - f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_overflow_to_wn_ids = [] - - subset_text = get_selection_text_for_in_statement(selected_ids) - - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_overflow_to_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_overflow_to_wn_ids = [] + + subset_text = get_selection_text_for_in_statement(selected_ids) + + # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_overflow_to_wn_ids.append(strrow) - logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") - logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") - return connected_overflow_to_wn_ids - - -# 10.12.2024 -def get_connected_we_to_re(subset_reaches): - """ - Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches - """ - - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_to_re_ids = [] - - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_to_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") + return connected_overflow_to_wn_ids + + + # 10.12.2024 + def get_connected_we_to_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_to_re_ids.append(strrow) - logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") - return connected_wn_to_re_ids - - -def get_ws_wn_ids(classname): - """ - Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) - """ - - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - ws_wn_ids = [] - - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - ws_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_to_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( - f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_wn_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") - - return ws_wn_ids - - -def get_ws_selected_ww_networkelements(selected_wwn): - """ - Get list of id's of wastewater_structure from selected wastewater_network_elements - """ - - logger.debug( - f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - selection_text = "" - - for list_item in selected_wwn: - selection_text += "'" - selection_text += list_item - selection_text += "'," - - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] - - logger.debug(f"selection_text = {selection_text} ...") - - ws_ids = [] - - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - ws_ids = None - else: - records = cursor.fetchall() - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") - - return ws_ids - - -# 10.1.2024 -def filter_reaches(selected_ids): - """ - Filter out reaches from selected_ids - """ - - logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") - - if selected_ids is None: - subset_reaches_ids = None - else: + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") + return connected_wn_to_re_ids + + + def get_ws_wn_ids(classname): + """ + Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) + """ + + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() - subset_reaches_ids = [] - all_reaches_ids = [] + ws_wn_ids = [] - get_selection_text_for_in_statement(selected_ids) - - # select all reaches - cursor.execute("SELECT obj_id FROM qgep_od.reach;") + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ # ws_wn_ids_count = int(cursor.fetchone()[0]) # if ws_wn_ids_count == 0: if cursor.fetchone() is None: - all_reaches_ids = None + ws_wn_ids = None else: # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute("SELECT obj_id FROM qgep_od.reach;") + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) records = cursor.fetchall() # 15.11.2024 - does not get all records, but only n-1 @@ -460,82 +365,177 @@ def filter_reaches(selected_ids): # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: - all_reaches_ids.append(strrow) - logger.debug(f" building up '{all_reaches_ids}' ...") + ws_wn_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_wn_ids + + + def get_ws_selected_ww_networkelements(selected_wwn): + """ + Get list of id's of wastewater_structure from selected wastewater_network_elements + """ + + logger.debug( + f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + selection_text = "" + + for list_item in selected_wwn: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] - for list_item in selected_ids: - if list_item in all_reaches_ids: - subset_reaches_ids.append(list_item) + logger.debug(f"selection_text = {selection_text} ...") + + ws_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_ids + + + # 10.1.2024 + def filter_reaches(selected_ids): + """ + Filter out reaches from selected_ids + """ + + logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") + + if selected_ids is None: + subset_reaches_ids = None + else: + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + subset_reaches_ids = [] + all_reaches_ids = [] + + get_selection_text_for_in_statement(selected_ids) + + # select all reaches + cursor.execute("SELECT obj_id FROM qgep_od.reach;") + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + all_reaches_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute("SELECT obj_id FROM qgep_od.reach;") + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + all_reaches_ids.append(strrow) + logger.debug(f" building up '{all_reaches_ids}' ...") + + for list_item in selected_ids: + if list_item in all_reaches_ids: + subset_reaches_ids.append(list_item) + logger.debug( + f"'filter_reaches: {list_item}' is a reach id - added to subset_reaches_ids" + ) + else: + logger.debug(f"'filter_reaches: {list_item}' is not a reach id") + logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") + return subset_reaches_ids + + + def remove_from_selection(selected_ids, remove_ids): + """ + Remove ids from selected_ids if they are in selected_ids + """ + + if remove_ids is None: + return selected_ids + else: + for list_item in remove_ids: + # selected_ids = selected_ids.remove(list_item) + try: + selected_ids.remove(list_item) + except Exception: logger.debug( - f"'filter_reaches: {list_item}' is a reach id - added to subset_reaches_ids" + f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" ) - else: - logger.debug(f"'filter_reaches: {list_item}' is not a reach id") - logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") - return subset_reaches_ids + return selected_ids -def remove_from_selection(selected_ids, remove_ids): - """ - Remove ids from selected_ids if they are in selected_ids - """ - if remove_ids is None: - return selected_ids - else: - for list_item in remove_ids: - # selected_ids = selected_ids.remove(list_item) - try: - selected_ids.remove(list_item) - except Exception: - logger.debug( - f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" - ) + def add_to_selection(selected_ids, add_ids): + """ + Append ids to selected_ids + """ - return selected_ids + if selected_ids is None: + selected_ids = [] + for list_item in add_ids: + # selected_ids = selected_ids.append(list_item) + selected_ids.append(list_item) -def add_to_selection(selected_ids, add_ids): - """ - Append ids to selected_ids - """ - - if selected_ids is None: - selected_ids = [] - - for list_item in add_ids: - # selected_ids = selected_ids.append(list_item) - selected_ids.append(list_item) - - return selected_ids - - def create_metaattributes(self, row): - metaattribute = self.abwasser_model.metaattribute( - # FIELDS TO MAP TO ABWASSER.metaattribute - # --- metaattribute --- - # 31.3.2023 identifier instead of name - # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? - # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? - # datenherr=getattr(row.fk_dataowner__REL, "identifier", "unknown"), # TODO : is unknown ok ? - # datenlieferant=getattr(row.fk_provider__REL, "identifier", "unknown"), # TODO : is unknown ok ? - # 31.3.2023 obj_id instead of name - datenherr=getattr( - row.fk_dataowner__REL, "obj_id", "unknown" - ), # TODO : is unknown ok ? - datenlieferant=getattr( - row.fk_provider__REL, "obj_id", "unknown" - ), # TODO : is unknown ok ? - letzte_aenderung=row.last_modification, - sia405_baseclass_metaattribute=self.get_tid(row), - # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row - t_id=self.get_tid(row), - t_seq=0, - ) + return selected_ids - if self.current_basket is not None: - metaattribute["t_basket"] = self.current_basket.t_id + def create_metaattributes(self, row): + metaattribute = self.abwasser_model.metaattribute( + # FIELDS TO MAP TO ABWASSER.metaattribute + # --- metaattribute --- + # 31.3.2023 identifier instead of name + # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenherr=getattr(row.fk_dataowner__REL, "identifier", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "identifier", "unknown"), # TODO : is unknown ok ? + # 31.3.2023 obj_id instead of name + datenherr=getattr( + row.fk_dataowner__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + datenlieferant=getattr( + row.fk_provider__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + letzte_aenderung=row.last_modification, + sia405_baseclass_metaattribute=self.get_tid(row), + # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row + t_id=self.get_tid(row), + t_seq=0, + ) + + if self.current_basket is not None: + metaattribute["t_basket"] = self.current_basket.t_id - self.abwasser_session.add(metaattribute) + self.abwasser_session.add(metaattribute) def base_common(self, row, type_name): """ From a46d805c3a8de8823fbbe943976ac633319e0f6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:28:36 +0000 Subject: [PATCH 019/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index a1cdf6f7..058a9ccb 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -126,7 +126,6 @@ def check_fk_in_subsetid(self, subset, relation): else: return self.tid_maker.tid_for_row(relation) - # 10.12.2024 def get_selection_text_for_in_statement(selection_list): """ @@ -145,7 +144,6 @@ def get_selection_text_for_in_statement(selection_list): logger.debug(f"selection_text = {selection_text} ...") return selection_text - # 12.11.2024 to clean up - get_ws_wn_ids kann das auch def get_cl_re_ids(classname): """ @@ -187,7 +185,6 @@ def get_cl_re_ids(classname): logger.warning(f"Do not use this function with {classname} !") return None - # 10.12.2024 def get_connected_we_from_re(subset_reaches): """ @@ -234,7 +231,6 @@ def get_connected_we_from_re(subset_reaches): logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") return connected_wn_from_re_ids - # 10.12.2024 def get_connected_overflow_to_wn_ids(selected_ids): """ @@ -281,7 +277,6 @@ def get_connected_overflow_to_wn_ids(selected_ids): logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") return connected_overflow_to_wn_ids - # 10.12.2024 def get_connected_we_to_re(subset_reaches): """ @@ -328,7 +323,6 @@ def get_connected_we_to_re(subset_reaches): logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids - def get_ws_wn_ids(classname): """ Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) @@ -370,7 +364,6 @@ def get_ws_wn_ids(classname): return ws_wn_ids - def get_ws_selected_ww_networkelements(selected_wwn): """ Get list of id's of wastewater_structure from selected wastewater_network_elements @@ -419,7 +412,6 @@ def get_ws_selected_ww_networkelements(selected_wwn): return ws_ids - # 10.1.2024 def filter_reaches(selected_ids): """ @@ -474,7 +466,6 @@ def filter_reaches(selected_ids): logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") return subset_reaches_ids - def remove_from_selection(selected_ids, remove_ids): """ Remove ids from selected_ids if they are in selected_ids @@ -494,7 +485,6 @@ def remove_from_selection(selected_ids, remove_ids): return selected_ids - def add_to_selection(selected_ids, add_ids): """ Append ids to selected_ids From 5c98da9bf40c9276491fac6dfdcc23b624b7d0ba Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:28:40 +0100 Subject: [PATCH 020/245] comment out import add_selection etc. --- qgepqwat2ili/qgepsia405/export.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index be329fd9..9e21886e 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -11,14 +11,14 @@ # ) from ..utils.qgep_export_utils import ( QgepExportUtils, - add_to_selection, - filter_reaches, - get_connected_overflow_to_wn_ids, - get_connected_we_from_re, - get_connected_we_to_re, - get_ws_selected_ww_networkelements, - get_ws_wn_ids, - remove_from_selection, + # add_to_selection, + # filter_reaches, + # get_connected_overflow_to_wn_ids, + # get_connected_we_from_re, + # get_connected_we_to_re, + # get_ws_selected_ww_networkelements, + # get_ws_wn_ids, + # remove_from_selection, ) from ..utils.various import logger from .model_abwasser import get_abwasser_model From c0c6a6faa26f039a20c37826e07cb2ba2b9d0043 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:30:57 +0000 Subject: [PATCH 021/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9e21886e..41cf019a 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -9,16 +9,8 @@ # 12.12.2024 - verschoben # ) -from ..utils.qgep_export_utils import ( +from ..utils.qgep_export_utils import ( # add_to_selection,; filter_reaches,; get_connected_overflow_to_wn_ids,; get_connected_we_from_re,; get_connected_we_to_re,; get_ws_selected_ww_networkelements,; get_ws_wn_ids,; remove_from_selection, QgepExportUtils, - # add_to_selection, - # filter_reaches, - # get_connected_overflow_to_wn_ids, - # get_connected_we_from_re, - # get_connected_we_to_re, - # get_ws_selected_ww_networkelements, - # get_ws_wn_ids, - # remove_from_selection, ) from ..utils.various import logger from .model_abwasser import get_abwasser_model From 333b7c04dc082d7f0c1681bb596ffc943963bc89 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:34:48 +0100 Subject: [PATCH 022/245] add qgep_export_utils part 1 --- qgepqwat2ili/qgepsia405/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9e21886e..f42f5720 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -73,11 +73,11 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske flag_approach_urs = True if flag_approach_urs: # 2. Get all connected from wastewater_nodes of selected reaches - connected_from_wn_ids = get_connected_we_from_re(subset_ids) + connected_from_wn_ids = qgep_export_utils.get_connected_we_from_re(subset_ids) # 3. Get all connected to wastewater_nodes of selected reaches - connected_to_wn_ids = get_connected_we_to_re(subset_ids) + connected_to_wn_ids = qgep_export_utils.get_connected_we_to_re(subset_ids) # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + connected_overflow_to_wn_ids = qgep_export_utils.get_connected_overflow_to_wn_ids(subset_ids) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) From cb81d24af1cf8aa23d7295ac6a40d82a84cd6258 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:35:18 +0000 Subject: [PATCH 023/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 588f259b..b62e5607 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -69,7 +69,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 3. Get all connected to wastewater_nodes of selected reaches connected_to_wn_ids = qgep_export_utils.get_connected_we_to_re(subset_ids) # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = qgep_export_utils.get_connected_overflow_to_wn_ids(subset_ids) + connected_overflow_to_wn_ids = qgep_export_utils.get_connected_overflow_to_wn_ids( + subset_ids + ) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) From 607749397f50ba5b411ca52f6142e033acd718f6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:44:21 +0100 Subject: [PATCH 024/245] take out functions out of class QgepExportUtils again --- qgepqwat2ili/qgepsia405/export.py | 3 +- qgepqwat2ili/utils/qgep_export_utils.py | 797 ++++++++++++------------ 2 files changed, 402 insertions(+), 398 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 588f259b..852f7035 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -9,8 +9,9 @@ # 12.12.2024 - verschoben # ) -from ..utils.qgep_export_utils import ( # add_to_selection,; filter_reaches,; get_connected_overflow_to_wn_ids,; get_connected_we_from_re,; get_connected_we_to_re,; get_ws_selected_ww_networkelements,; get_ws_wn_ids,; remove_from_selection, +from ..utils.qgep_export_utils import ( QgepExportUtils, + add_to_selection,; filter_reaches,; get_connected_overflow_to_wn_ids,; get_connected_we_from_re,; get_connected_we_to_re,; get_ws_selected_ww_networkelements,; get_ws_wn_ids,; remove_from_selection, ) from ..utils.various import logger from .model_abwasser import get_abwasser_model diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 058a9ccb..c25c793b 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -126,406 +126,33 @@ def check_fk_in_subsetid(self, subset, relation): else: return self.tid_maker.tid_for_row(relation) - # 10.12.2024 - def get_selection_text_for_in_statement(selection_list): - """ - convert selection_list to selection_text to fit SQL IN statement - """ - selection_text = "" - - for list_item in selection_list: - selection_text += "'" - selection_text += list_item - selection_text += "'," - - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] - - logger.debug(f"selection_text = {selection_text} ...") - return selection_text - - # 12.11.2024 to clean up - get_ws_wn_ids kann das auch - def get_cl_re_ids(classname): - """ - Get list of id's of reaches of the channels provided - """ - - # define classes that this is allowed to use - adapt for TWW to include model changes - if classname == "channel": - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") - - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - cl_re_ids = [] - - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # cl_re_ids_count = int(cursor.fetchone()[0]) - # if cl_re_ids_count == 0: - if cursor.fetchone() is None: - cl_re_ids = None - else: - records = cursor.fetchall() - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - cl_re_ids.append(strrow) - # logger.debug(f" building up '{cl_re_ids}' ...") - - logger.info(f" cl_re_ids: '{cl_re_ids}'") - return cl_re_ids - else: - logger.warning(f"Do not use this function with {classname} !") - return None - - # 10.12.2024 - def get_connected_we_from_re(subset_reaches): - """ - Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches - """ - - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_from_re_ids = [] - - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - - # select all connected from wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_from_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_from_re_ids.append(strrow) - logger.debug(f" building up '{connected_wn_from_re_ids}' ...") - logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") - return connected_wn_from_re_ids - - # 10.12.2024 - def get_connected_overflow_to_wn_ids(selected_ids): - """ - Get all connected wastewater_nodes from overflows.fk_overflow_to - """ - - logger.info( - f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_overflow_to_wn_ids = [] - - subset_text = get_selection_text_for_in_statement(selected_ids) - - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_overflow_to_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_overflow_to_wn_ids.append(strrow) - logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") - logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") - return connected_overflow_to_wn_ids - - # 10.12.2024 - def get_connected_we_to_re(subset_reaches): - """ - Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches - """ - - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + def create_metaattributes(self, row): + metaattribute = self.abwasser_model.metaattribute( + # FIELDS TO MAP TO ABWASSER.metaattribute + # --- metaattribute --- + # 31.3.2023 identifier instead of name + # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenherr=getattr(row.fk_dataowner__REL, "identifier", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "identifier", "unknown"), # TODO : is unknown ok ? + # 31.3.2023 obj_id instead of name + datenherr=getattr( + row.fk_dataowner__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + datenlieferant=getattr( + row.fk_provider__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + letzte_aenderung=row.last_modification, + sia405_baseclass_metaattribute=self.get_tid(row), + # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row + t_id=self.get_tid(row), + t_seq=0, ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_to_re_ids = [] - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_to_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_to_re_ids.append(strrow) - logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") - return connected_wn_to_re_ids - - def get_ws_wn_ids(classname): - """ - Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) - """ - - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - ws_wn_ids = [] - - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - ws_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_wn_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") - - return ws_wn_ids - - def get_ws_selected_ww_networkelements(selected_wwn): - """ - Get list of id's of wastewater_structure from selected wastewater_network_elements - """ - - logger.debug( - f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - selection_text = "" - - for list_item in selected_wwn: - selection_text += "'" - selection_text += list_item - selection_text += "'," - - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] - - logger.debug(f"selection_text = {selection_text} ...") - - ws_ids = [] - - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - ws_ids = None - else: - records = cursor.fetchall() - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") - - return ws_ids - - # 10.1.2024 - def filter_reaches(selected_ids): - """ - Filter out reaches from selected_ids - """ - - logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") - - if selected_ids is None: - subset_reaches_ids = None - else: - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - subset_reaches_ids = [] - all_reaches_ids = [] - - get_selection_text_for_in_statement(selected_ids) - - # select all reaches - cursor.execute("SELECT obj_id FROM qgep_od.reach;") - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - all_reaches_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute("SELECT obj_id FROM qgep_od.reach;") - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - all_reaches_ids.append(strrow) - logger.debug(f" building up '{all_reaches_ids}' ...") - - for list_item in selected_ids: - if list_item in all_reaches_ids: - subset_reaches_ids.append(list_item) - logger.debug( - f"'filter_reaches: {list_item}' is a reach id - added to subset_reaches_ids" - ) - else: - logger.debug(f"'filter_reaches: {list_item}' is not a reach id") - logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") - return subset_reaches_ids - - def remove_from_selection(selected_ids, remove_ids): - """ - Remove ids from selected_ids if they are in selected_ids - """ - - if remove_ids is None: - return selected_ids - else: - for list_item in remove_ids: - # selected_ids = selected_ids.remove(list_item) - try: - selected_ids.remove(list_item) - except Exception: - logger.debug( - f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" - ) - - return selected_ids - - def add_to_selection(selected_ids, add_ids): - """ - Append ids to selected_ids - """ - - if selected_ids is None: - selected_ids = [] - - for list_item in add_ids: - # selected_ids = selected_ids.append(list_item) - selected_ids.append(list_item) - - return selected_ids - - def create_metaattributes(self, row): - metaattribute = self.abwasser_model.metaattribute( - # FIELDS TO MAP TO ABWASSER.metaattribute - # --- metaattribute --- - # 31.3.2023 identifier instead of name - # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? - # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? - # datenherr=getattr(row.fk_dataowner__REL, "identifier", "unknown"), # TODO : is unknown ok ? - # datenlieferant=getattr(row.fk_provider__REL, "identifier", "unknown"), # TODO : is unknown ok ? - # 31.3.2023 obj_id instead of name - datenherr=getattr( - row.fk_dataowner__REL, "obj_id", "unknown" - ), # TODO : is unknown ok ? - datenlieferant=getattr( - row.fk_provider__REL, "obj_id", "unknown" - ), # TODO : is unknown ok ? - letzte_aenderung=row.last_modification, - sia405_baseclass_metaattribute=self.get_tid(row), - # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row - t_id=self.get_tid(row), - t_seq=0, - ) - - if self.current_basket is not None: - metaattribute["t_basket"] = self.current_basket.t_id + if self.current_basket is not None: + metaattribute["t_basket"] = self.current_basket.t_id - self.abwasser_session.add(metaattribute) + self.abwasser_session.add(metaattribute) def base_common(self, row, type_name): """ @@ -1124,3 +751,379 @@ def export_benching(self): print(".", end="") logger.info("done") self.abwasser_session.flush() + +#end class QgepExportUtils + +# 10.12.2024 +def get_selection_text_for_in_statement(selection_list): + """ + convert selection_list to selection_text to fit SQL IN statement + """ + selection_text = "" + + for list_item in selection_list: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] + + logger.debug(f"selection_text = {selection_text} ...") + return selection_text + +# 12.11.2024 to clean up - get_ws_wn_ids kann das auch +def get_cl_re_ids(classname): + """ + Get list of id's of reaches of the channels provided + """ + + # define classes that this is allowed to use - adapt for TWW to include model changes + if classname == "channel": + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + cl_re_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # cl_re_ids_count = int(cursor.fetchone()[0]) + # if cl_re_ids_count == 0: + if cursor.fetchone() is None: + cl_re_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + cl_re_ids.append(strrow) + # logger.debug(f" building up '{cl_re_ids}' ...") + + logger.info(f" cl_re_ids: '{cl_re_ids}'") + return cl_re_ids + else: + logger.warning(f"Do not use this function with {classname} !") + return None + +# 10.12.2024 +def get_connected_we_from_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_from_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected from wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_from_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_from_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") + return connected_wn_from_re_ids + +# 10.12.2024 +def get_connected_overflow_to_wn_ids(selected_ids): + """ + Get all connected wastewater_nodes from overflows.fk_overflow_to + """ + + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_overflow_to_wn_ids = [] + + subset_text = get_selection_text_for_in_statement(selected_ids) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") + return connected_overflow_to_wn_ids + +# 10.12.2024 +def get_connected_we_to_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_to_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") + return connected_wn_to_re_ids + +def get_ws_wn_ids(classname): + """ + Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) + """ + + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + ws_wn_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_wn_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_wn_ids + +def get_ws_selected_ww_networkelements(selected_wwn): + """ + Get list of id's of wastewater_structure from selected wastewater_network_elements + """ + + logger.debug( + f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + selection_text = "" + + for list_item in selected_wwn: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] + + logger.debug(f"selection_text = {selection_text} ...") + + ws_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_ids + +# 10.1.2024 +def filter_reaches(selected_ids): + """ + Filter out reaches from selected_ids + """ + + logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") + + if selected_ids is None: + subset_reaches_ids = None + else: + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + subset_reaches_ids = [] + all_reaches_ids = [] + + get_selection_text_for_in_statement(selected_ids) + + # select all reaches + cursor.execute("SELECT obj_id FROM qgep_od.reach;") + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + all_reaches_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute("SELECT obj_id FROM qgep_od.reach;") + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + all_reaches_ids.append(strrow) + logger.debug(f" building up '{all_reaches_ids}' ...") + + for list_item in selected_ids: + if list_item in all_reaches_ids: + subset_reaches_ids.append(list_item) + logger.debug( + f"'filter_reaches: {list_item}' is a reach id - added to subset_reaches_ids" + ) + else: + logger.debug(f"'filter_reaches: {list_item}' is not a reach id") + logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") + return subset_reaches_ids + +def remove_from_selection(selected_ids, remove_ids): + """ + Remove ids from selected_ids if they are in selected_ids + """ + + if remove_ids is None: + return selected_ids + else: + for list_item in remove_ids: + # selected_ids = selected_ids.remove(list_item) + try: + selected_ids.remove(list_item) + except Exception: + logger.debug( + f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" + ) + + return selected_ids + +def add_to_selection(selected_ids, add_ids): + """ + Append ids to selected_ids + """ + + if selected_ids is None: + selected_ids = [] + + for list_item in add_ids: + # selected_ids = selected_ids.append(list_item) + selected_ids.append(list_item) + + return selected_ids + From ce215b52bf97ac22d171dd46f2b49b27c9a6993b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:44:49 +0000 Subject: [PATCH 025/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 12 ++++++++++-- qgepqwat2ili/utils/qgep_export_utils.py | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 0ee0a010..97fab4ca 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -9,9 +9,17 @@ # 12.12.2024 - verschoben # ) -from ..utils.qgep_export_utils import ( +from ..utils.qgep_export_utils import ( QgepExportUtils, - add_to_selection,; filter_reaches,; get_connected_overflow_to_wn_ids,; get_connected_we_from_re,; get_connected_we_to_re,; get_ws_selected_ww_networkelements,; get_ws_wn_ids,; remove_from_selection, + ;, + add_to_selection, + filter_reaches, + get_connected_overflow_to_wn_ids, + get_connected_we_from_re, + get_connected_we_to_re, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, ) from ..utils.various import logger from .model_abwasser import get_abwasser_model diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index c25c793b..439c8d66 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -752,7 +752,9 @@ def export_benching(self): logger.info("done") self.abwasser_session.flush() -#end class QgepExportUtils + +# end class QgepExportUtils + # 10.12.2024 def get_selection_text_for_in_statement(selection_list): @@ -772,6 +774,7 @@ def get_selection_text_for_in_statement(selection_list): logger.debug(f"selection_text = {selection_text} ...") return selection_text + # 12.11.2024 to clean up - get_ws_wn_ids kann das auch def get_cl_re_ids(classname): """ @@ -813,6 +816,7 @@ def get_cl_re_ids(classname): logger.warning(f"Do not use this function with {classname} !") return None + # 10.12.2024 def get_connected_we_from_re(subset_reaches): """ @@ -859,6 +863,7 @@ def get_connected_we_from_re(subset_reaches): logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") return connected_wn_from_re_ids + # 10.12.2024 def get_connected_overflow_to_wn_ids(selected_ids): """ @@ -905,6 +910,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") return connected_overflow_to_wn_ids + # 10.12.2024 def get_connected_we_to_re(subset_reaches): """ @@ -951,6 +957,7 @@ def get_connected_we_to_re(subset_reaches): logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids + def get_ws_wn_ids(classname): """ Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) @@ -992,6 +999,7 @@ def get_ws_wn_ids(classname): return ws_wn_ids + def get_ws_selected_ww_networkelements(selected_wwn): """ Get list of id's of wastewater_structure from selected wastewater_network_elements @@ -1040,6 +1048,7 @@ def get_ws_selected_ww_networkelements(selected_wwn): return ws_ids + # 10.1.2024 def filter_reaches(selected_ids): """ @@ -1094,6 +1103,7 @@ def filter_reaches(selected_ids): logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") return subset_reaches_ids + def remove_from_selection(selected_ids, remove_ids): """ Remove ids from selected_ids if they are in selected_ids @@ -1113,6 +1123,7 @@ def remove_from_selection(selected_ids, remove_ids): return selected_ids + def add_to_selection(selected_ids, add_ids): """ Append ids to selected_ids @@ -1126,4 +1137,3 @@ def add_to_selection(selected_ids, add_ids): selected_ids.append(list_item) return selected_ids - From 5d68bb9b3e7d20d924c30b784c9015b83ca0718e Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:46:13 +0100 Subject: [PATCH 026/245] adaptions import --- qgepqwat2ili/qgepsia405/export.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 0ee0a010..2b64cc6b 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -11,7 +11,14 @@ # ) from ..utils.qgep_export_utils import ( QgepExportUtils, - add_to_selection,; filter_reaches,; get_connected_overflow_to_wn_ids,; get_connected_we_from_re,; get_connected_we_to_re,; get_ws_selected_ww_networkelements,; get_ws_wn_ids,; remove_from_selection, + add_to_selection, + filter_reaches, + get_connected_overflow_to_wn_ids, + get_connected_we_from_re, + get_connected_we_to_re, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, ) from ..utils.various import logger from .model_abwasser import get_abwasser_model From b822e3387e333d37a397e01fb31bd57c3e8f1e3a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:47:10 +0000 Subject: [PATCH 027/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index d0ae505e..da320d5c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -13,9 +13,6 @@ QgepExportUtils, add_to_selection, filter_reaches, - get_connected_overflow_to_wn_ids, - get_connected_we_from_re, - get_connected_we_to_re, get_ws_selected_ww_networkelements, get_ws_wn_ids, remove_from_selection, From 672564a6fd3255646836873589cbe74fab2abe14 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:48:01 +0100 Subject: [PATCH 028/245] take out qgep_export_utils. --- qgepqwat2ili/qgepsia405/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index d0ae505e..a1aac03e 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -73,11 +73,11 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske flag_approach_urs = True if flag_approach_urs: # 2. Get all connected from wastewater_nodes of selected reaches - connected_from_wn_ids = qgep_export_utils.get_connected_we_from_re(subset_ids) + connected_from_wn_ids = get_connected_we_from_re(subset_ids) # 3. Get all connected to wastewater_nodes of selected reaches - connected_to_wn_ids = qgep_export_utils.get_connected_we_to_re(subset_ids) + connected_to_wn_ids = get_connected_we_to_re(subset_ids) # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = qgep_export_utils.get_connected_overflow_to_wn_ids( + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids( subset_ids ) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids From ef4f473a0ade7f3c3cefc5e6d98807b0231096a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:48:29 +0000 Subject: [PATCH 029/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 3e4fef08..45f0158c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -74,9 +74,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 3. Get all connected to wastewater_nodes of selected reaches connected_to_wn_ids = get_connected_we_to_re(subset_ids) # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids( - subset_ids - ) + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) From 1d9641901e4615839fe6cb1de5550d1a3fd17cf4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 10:50:22 +0100 Subject: [PATCH 030/245] readd get_connected_overflow_to_wn_ids etc. --- qgepqwat2ili/qgepsia405/export.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 3e4fef08..4b8401d7 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -13,6 +13,9 @@ QgepExportUtils, add_to_selection, filter_reaches, + get_connected_we_from_re, + get_connected_we_to_re, + get_connected_overflow_to_wn_ids, get_ws_selected_ww_networkelements, get_ws_wn_ids, remove_from_selection, From 5686c44572f8145f7888a501d72cd58045926509 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:50:44 +0000 Subject: [PATCH 031/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 0a6b87e6..be329fd9 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -13,9 +13,9 @@ QgepExportUtils, add_to_selection, filter_reaches, + get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, - get_connected_overflow_to_wn_ids, get_ws_selected_ww_networkelements, get_ws_wn_ids, remove_from_selection, From 730c63aba38065fa0bb4a028b174dff4a855ff8b Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:11:05 +0100 Subject: [PATCH 032/245] adapt filter dryweather_downspout (as in tww) --- qgepqwat2ili/utils/qgep_export_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 439c8d66..f4ed9c24 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -561,9 +561,9 @@ def export_reach(self): def export_dryweather_downspout(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) if self.filtered: - query = query.join( - self.qgep_model.wastewater_structure, self.qgep_model.wastewater_networkelement - ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = query.join(self.qgep_model.wastewater_structure) + .join(self.qgep_model.wastewater_networkelement) + .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout From cb41389abac70368e1738de3ab720232b51d21f3 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:13:47 +0100 Subject: [PATCH 033/245] reformat intend --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index f4ed9c24..ef4d3b51 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -562,7 +562,7 @@ def export_dryweather_downspout(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) if self.filtered: query = query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) + .join(self.qgep_model.wastewater_networkelement) .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout From aa84dab79e232de86178ea8f02ff67495a1a8ac7 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:17:32 +0100 Subject: [PATCH 034/245] adapt ) --- qgepqwat2ili/utils/qgep_export_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index ef4d3b51..a794fc38 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -561,9 +561,13 @@ def export_reach(self): def export_dryweather_downspout(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) if self.filtered: - query = query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) - .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = ( + query.join(self.qgep_model.wastewater_structure) + .join(self.qgep_model.wastewater_networkelement) + .filter( + self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + ) + ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout From 9124d003685fd3f498965c06aaba7a953e7914c2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:17:46 +0000 Subject: [PATCH 035/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index a794fc38..96e3b65a 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -564,9 +564,7 @@ def export_dryweather_downspout(self): query = ( query.join(self.qgep_model.wastewater_structure) .join(self.qgep_model.wastewater_networkelement) - .filter( - self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) - ) + .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout From 6bc71639f0d063e89cd2c5f782a3db0f2e28fca0 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:20:50 +0100 Subject: [PATCH 036/245] same for other subclasses cover etc. --- qgepqwat2ili/utils/qgep_export_utils.py | 32 +++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 96e3b65a..dde99651 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -600,9 +600,11 @@ def export_dryweather_downspout(self): def export_access_aid(self): query = self.qgep_session.query(self.qgep_model.access_aid) if self.filtered: - query = query.join( - self.qgep_model.wastewater_structure, self.qgep_model.wastewater_networkelement - ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = ( + query.join(self.qgep_model.wastewater_structure) + .join(self.qgep_model.wastewater_networkelement) + .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.access_aid @@ -637,9 +639,11 @@ def export_access_aid(self): def export_dryweather_flume(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) if self.filtered: - query = query.join( - self.qgep_model.wastewater_structure, self.qgep_model.wastewater_networkelement - ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = ( + query.join(self.qgep_model.wastewater_structure) + .join(self.qgep_model.wastewater_networkelement) + .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_flume @@ -674,9 +678,11 @@ def export_dryweather_flume(self): def export_cover(self): query = self.qgep_session.query(self.qgep_model.cover) if self.filtered: - query = query.join( - self.qgep_model.wastewater_structure, self.qgep_model.wastewater_networkelement - ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = ( + query.join(self.qgep_model.wastewater_structure) + .join(self.qgep_model.wastewater_networkelement) + .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.cover @@ -720,9 +726,11 @@ def export_cover(self): def export_benching(self): query = self.qgep_session.query(self.qgep_model.benching) if self.filtered: - query = query.join( - self.qgep_model.wastewater_structure, self.qgep_model.wastewater_networkelement - ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = ( + query.join(self.qgep_model.wastewater_structure) + .join(self.qgep_model.wastewater_networkelement) + .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.benching From 2a8bd4900d679ebe5c511f212866445b96b201b6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:36:02 +0100 Subject: [PATCH 037/245] integrate subset_wws_ids --- qgepqwat2ili/utils/qgep_export_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index dde99651..df25dbf9 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -20,6 +20,7 @@ def __init__( labelorientation, filtered, subset_ids, + subset_wws_ids, ): self.tid_maker = tid_maker self.current_basket = current_basket @@ -30,6 +31,7 @@ def __init__( self.labelorientation = labelorientation self.filtered = filtered self.subset_ids = subset_ids + self.subset_wws_ids = subset_wws_ids def get_tid(self, relation): """ @@ -208,7 +210,11 @@ def wastewater_networkelement_common(self, row): Returns common attributes for wastewater_networkelement """ return { + # 12.12.2024 "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid( + self.subset_wws_ids, row.fk_wastewater_structure__REL + ), "bemerkung": self.truncate(self.emptystr_to_null(row.remark), 80), "bezeichnung": self.null_to_emptystr(row.identifier), } From 0b3aed052143ea46099f4f4913bb139d1e170981 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:41:15 +0100 Subject: [PATCH 038/245] take out duplicate --- qgepqwat2ili/utils/qgep_export_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index df25dbf9..fea651e4 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -210,8 +210,8 @@ def wastewater_networkelement_common(self, row): Returns common attributes for wastewater_networkelement """ return { - # 12.12.2024 - "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), + # added check_fk_in_subsetid with subset_wws_ids (only needed for SIA405 Abwasser export wwtp_structure - but as now in qgep_export_utils done for all export - might slow donw export + #"abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), "abwasserbauwerkref": check_fk_in_subsetid( self.subset_wws_ids, row.fk_wastewater_structure__REL ), From bf09e522725ff46e7aa78e740c2c670a39c2a84d Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:42:00 +0100 Subject: [PATCH 039/245] added self. --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index fea651e4..d01f3088 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -212,7 +212,7 @@ def wastewater_networkelement_common(self, row): return { # added check_fk_in_subsetid with subset_wws_ids (only needed for SIA405 Abwasser export wwtp_structure - but as now in qgep_export_utils done for all export - might slow donw export #"abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), - "abwasserbauwerkref": check_fk_in_subsetid( + "abwasserbauwerkref": self.check_fk_in_subsetid( self.subset_wws_ids, row.fk_wastewater_structure__REL ), "bemerkung": self.truncate(self.emptystr_to_null(row.remark), 80), From 4a7512f57fa2897da6da0e25cee29d3469b3fd6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:42:40 +0000 Subject: [PATCH 040/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d01f3088..276586a2 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -211,7 +211,7 @@ def wastewater_networkelement_common(self, row): """ return { # added check_fk_in_subsetid with subset_wws_ids (only needed for SIA405 Abwasser export wwtp_structure - but as now in qgep_export_utils done for all export - might slow donw export - #"abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), + # "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), "abwasserbauwerkref": self.check_fk_in_subsetid( self.subset_wws_ids, row.fk_wastewater_structure__REL ), From be5957fce0f40371e9e338f57e3ed68fa9b797bf Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 11:49:05 +0100 Subject: [PATCH 041/245] initialize subset_wws_ids --- qgepqwat2ili/qgepsia405/export.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index be329fd9..897060ec 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -69,6 +69,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] + subset_wws_ids = [] flag_approach_urs = True if flag_approach_urs: @@ -188,6 +189,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske labelorientation=labelorientation, filtered=filtered, subset_ids=subset_ids, + subset_wws_ids=subset_wws_ids, ) # ADAPTED FROM 052a_sia405_abwasser_2015_2_d_interlisexport2.sql From dae92ef031569cd937c4cbb51853b1ac59b0d707 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 13:04:31 +0100 Subject: [PATCH 042/245] take out some logger.debug --- qgepqwat2ili/utils/qgep_export_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 276586a2..f8d51b52 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -875,7 +875,7 @@ def get_connected_we_from_re(subset_reaches): strrow = str(row[0]) if strrow is not None: connected_wn_from_re_ids.append(strrow) - logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + # logger.debug(f" building up '{connected_wn_from_re_ids}' ...") logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") return connected_wn_from_re_ids @@ -922,7 +922,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): strrow = str(row[0]) if strrow is not None: connected_overflow_to_wn_ids.append(strrow) - logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") return connected_overflow_to_wn_ids @@ -969,7 +969,7 @@ def get_connected_we_to_re(subset_reaches): strrow = str(row[0]) if strrow is not None: connected_wn_to_re_ids.append(strrow) - logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids @@ -1106,7 +1106,7 @@ def filter_reaches(selected_ids): strrow = str(row[0]) if strrow is not None: all_reaches_ids.append(strrow) - logger.debug(f" building up '{all_reaches_ids}' ...") + # logger.debug(f" building up '{all_reaches_ids}' ...") for list_item in selected_ids: if list_item in all_reaches_ids: From 8603590d74e39a29ab7873eb0a8172bb978956f0 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 13:39:53 +0100 Subject: [PATCH 043/245] def get_ws_ids and adapt code --- qgepqwat2ili/qgepsia405/export.py | 2 +- qgepqwat2ili/utils/qgep_export_utils.py | 49 +++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 897060ec..13f849ca 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -116,7 +116,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ) # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") # 10. Show ws_off_sia405abwasser_list logger.info( diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index f8d51b52..d957b7ec 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -899,7 +899,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ @@ -911,7 +911,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" ) records = cursor.fetchall() @@ -946,7 +946,7 @@ def get_connected_we_to_re(subset_reaches): # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ @@ -958,7 +958,7 @@ def get_connected_we_to_re(subset_reaches): # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" ) records = cursor.fetchall() @@ -1015,6 +1015,47 @@ def get_ws_wn_ids(classname): return ws_wn_ids +#12.12.2024 +def get_ws_ids(classname): + """ + Get list of id's of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) + """ + + logger.info(f"get list of id's of subclass {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + ws_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.{classname};" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.{classname};" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_ids def get_ws_selected_ww_networkelements(selected_wwn): """ From a514128a1cbbcb417217e39bfe84b1434477264b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:40:10 +0000 Subject: [PATCH 044/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d957b7ec..65f1f5eb 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1015,7 +1015,8 @@ def get_ws_wn_ids(classname): return ws_wn_ids -#12.12.2024 + +# 12.12.2024 def get_ws_ids(classname): """ Get list of id's of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) @@ -1029,21 +1030,17 @@ def get_ws_ids(classname): ws_ids = [] # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT ws.obj_id FROM qgep_od.{classname};" - ) + cursor.execute(f"SELECT ws.obj_id FROM qgep_od.{classname};") # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ # ws_wn_ids_count = int(cursor.fetchone()[0]) # if ws_wn_ids_count == 0: if cursor.fetchone() is None: - ws_wn_ids = None + pass else: # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT ws.obj_id FROM qgep_od.{classname};" - ) + cursor.execute(f"SELECT ws.obj_id FROM qgep_od.{classname};") records = cursor.fetchall() # 15.11.2024 - does not get all records, but only n-1 @@ -1057,6 +1054,7 @@ def get_ws_ids(classname): return ws_ids + def get_ws_selected_ww_networkelements(selected_wwn): """ Get list of id's of wastewater_structure from selected wastewater_network_elements From a2aa4e898acda37f30e94dbe72f9cdbe2875aba3 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 13:41:12 +0100 Subject: [PATCH 045/245] import get_ws_ids --- qgepqwat2ili/qgepsia405/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 13f849ca..ab07a06d 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -17,6 +17,7 @@ get_connected_we_from_re, get_connected_we_to_re, get_ws_selected_ww_networkelements, + get_ws_ids, get_ws_wn_ids, remove_from_selection, ) From feca9f9fb0b16e74d82d7a0b8778781717043468 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:41:26 +0000 Subject: [PATCH 046/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index ab07a06d..fa9c7e22 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -16,8 +16,8 @@ get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, - get_ws_selected_ww_networkelements, get_ws_ids, + get_ws_selected_ww_networkelements, get_ws_wn_ids, remove_from_selection, ) From 988c2c1773a753000b147230fd14888761dc853f Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 13:48:28 +0100 Subject: [PATCH 047/245] correct query --- qgepqwat2ili/utils/qgep_export_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 65f1f5eb..536ba457 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1030,7 +1030,7 @@ def get_ws_ids(classname): ws_ids = [] # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute(f"SELECT ws.obj_id FROM qgep_od.{classname};") + cursor.execute(f"SELECT obj_id FROM qgep_od.{classname};") # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ # ws_wn_ids_count = int(cursor.fetchone()[0]) @@ -1040,7 +1040,7 @@ def get_ws_ids(classname): else: # added cursor.execute again to see if with this all records will be available # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute(f"SELECT ws.obj_id FROM qgep_od.{classname};") + cursor.execute(f"SELECT obj_id FROM qgep_od.{classname};") records = cursor.fetchall() # 15.11.2024 - does not get all records, but only n-1 From a8bacadb08c2bf6c0148bd3f036cbca6cee5e115 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 13:57:02 +0100 Subject: [PATCH 048/245] check on None for add_ids --- qgepqwat2ili/utils/qgep_export_utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 536ba457..20394160 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1183,12 +1183,14 @@ def add_to_selection(selected_ids, add_ids): """ Append ids to selected_ids """ + if add_ids is None: + return add_to_selection + else: + if selected_ids is None: + selected_ids = [] - if selected_ids is None: - selected_ids = [] - - for list_item in add_ids: - # selected_ids = selected_ids.append(list_item) - selected_ids.append(list_item) + for list_item in add_ids: + # selected_ids = selected_ids.append(list_item) + selected_ids.append(list_item) - return selected_ids + return selected_ids From 829e1c4fb336dbcb30718c21d3433d832c490138 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:03:00 +0100 Subject: [PATCH 049/245] add subset_wws_ids=subset_wws_ids, --- qgepqwat2ili/qgep/export.py | 1 + qgepqwat2ili/qgepdss/export.py | 1 + 2 files changed, 2 insertions(+) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 07087053..d49dda50 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -73,6 +73,7 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e labelorientation=labelorientation, filtered=filtered, subset_ids=subset_ids, + subset_wws_ids=subset_wws_ids, ) # ADAPTED FROM 052a_sia405_abwasser_2015_2_d_interlisexport2.sql diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 6bcfe72e..b2ad7757 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -80,6 +80,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e labelorientation=labelorientation, filtered=filtered, subset_ids=subset_ids, + subset_wws_ids=subset_wws_ids, ) def organisation_common(row): From 9ce7d634bc3350cf920776e81b38c6d4df36d555 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:11:41 +0100 Subject: [PATCH 050/245] functions check on None variables --- qgepqwat2ili/utils/qgep_export_utils.py | 445 +++++++++++------------- 1 file changed, 210 insertions(+), 235 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 20394160..8272f6d9 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -777,107 +777,70 @@ def get_selection_text_for_in_statement(selection_list): """ convert selection_list to selection_text to fit SQL IN statement """ - selection_text = "" + if selection_list is None: + selection_text = None + else: + selection_text = "" - for list_item in selection_list: - selection_text += "'" - selection_text += list_item - selection_text += "'," + for list_item in selection_list: + selection_text += "'" + selection_text += list_item + selection_text += "'," - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] - logger.debug(f"selection_text = {selection_text} ...") - return selection_text + logger.debug(f"selection_text = {selection_text} ...") + return selection_text -# 12.11.2024 to clean up - get_ws_wn_ids kann das auch -def get_cl_re_ids(classname): +# 10.12.2024 +def get_connected_we_from_re(subset_reaches): """ - Get list of id's of reaches of the channels provided + Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches """ - - # define classes that this is allowed to use - adapt for TWW to include model changes - if classname == "channel": - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") - + if subset_reaches is None: + connected_wn_from_re_ids = None + else: + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() - cl_re_ids = [] + connected_wn_from_re_ids = [] - # select all obj_id of the wastewater_nodes of wwtp_structure + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected from wastewater_nodes from provided subset of reaches cursor.execute( - "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # cl_re_ids_count = int(cursor.fetchone()[0]) - # if cl_re_ids_count == 0: + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: if cursor.fetchone() is None: - cl_re_ids = None + connected_wn_from_re_ids = None else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 for row in records: logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) - cl_re_ids.append(strrow) - # logger.debug(f" building up '{cl_re_ids}' ...") - - logger.info(f" cl_re_ids: '{cl_re_ids}'") - return cl_re_ids - else: - logger.warning(f"Do not use this function with {classname} !") - return None - - -# 10.12.2024 -def get_connected_we_from_re(subset_reaches): - """ - Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches - """ - - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_from_re_ids = [] - - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - - # select all connected from wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_from_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_from_re_ids.append(strrow) - # logger.debug(f" building up '{connected_wn_from_re_ids}' ...") - logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") - return connected_wn_from_re_ids + if strrow is not None: + connected_wn_from_re_ids.append(strrow) + # logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") + return connected_wn_from_re_ids # 10.12.2024 @@ -885,46 +848,48 @@ def get_connected_overflow_to_wn_ids(selected_ids): """ Get all connected wastewater_nodes from overflows.fk_overflow_to """ + if selected_ids is None: + connected_overflow_to_wn_ids = None + else: + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - logger.info( - f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_overflow_to_wn_ids = [] - - subset_text = get_selection_text_for_in_statement(selected_ids) + connected_overflow_to_wn_ids = [] - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" - ) + subset_text = get_selection_text_for_in_statement(selected_ids) - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_overflow_to_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" ) - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_overflow_to_wn_ids.append(strrow) - # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") - logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") - return connected_overflow_to_wn_ids + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") + return connected_overflow_to_wn_ids # 10.12.2024 @@ -932,88 +897,92 @@ def get_connected_we_to_re(subset_reaches): """ Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches """ + if subset_reaches is None: + connected_wn_to_re_ids = None + else: + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_to_re_ids = [] - - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + connected_wn_to_re_ids = [] - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" - ) + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_to_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" ) - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_to_re_ids.append(strrow) - # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") - return connected_wn_to_re_ids + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") + return connected_wn_to_re_ids def get_ws_wn_ids(classname): """ Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) """ + if classname is None: + ws_wn_ids = None + else: + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - ws_wn_ids = [] - - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" - ) + ws_wn_ids = [] - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - ws_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all obj_id of the wastewater_nodes of wwtp_structure cursor.execute( f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" ) - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_wn_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + records = cursor.fetchall() - return ws_wn_ids + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_wn_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_wn_ids # 12.12.2024 @@ -1022,37 +991,40 @@ def get_ws_ids(classname): Get list of id's of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) """ - logger.info(f"get list of id's of subclass {classname} ...") - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - ws_ids = [] + if classname is None: + ws_ids = None + else: + logger.info(f"get list of id's of subclass {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute(f"SELECT obj_id FROM qgep_od.{classname};") + ws_ids = [] - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - pass - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all obj_id of the wastewater_nodes of wwtp_structure cursor.execute(f"SELECT obj_id FROM qgep_od.{classname};") - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + pass + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute(f"SELECT obj_id FROM qgep_od.{classname};") + records = cursor.fetchall() - return ws_ids + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_ids def get_ws_selected_ww_networkelements(selected_wwn): @@ -1060,48 +1032,52 @@ def get_ws_selected_ww_networkelements(selected_wwn): Get list of id's of wastewater_structure from selected wastewater_network_elements """ - logger.debug( - f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() + if selected_wwn is None: + ws_ids = None + else: + + logger.debug( + f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - selection_text = "" + selection_text = "" - for list_item in selected_wwn: - selection_text += "'" - selection_text += list_item - selection_text += "'," + for list_item in selected_wwn: + selection_text += "'" + selection_text += list_item + selection_text += "'," - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] - logger.debug(f"selection_text = {selection_text} ...") + logger.debug(f"selection_text = {selection_text} ...") - ws_ids = [] + ws_ids = [] - # select all obj_id of the wastewater_nodes of wwtp_structure - cursor.execute( - f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" - ) + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" + ) - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - ws_ids = None - else: - records = cursor.fetchall() - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - ws_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") - return ws_ids + return ws_ids # 10.1.2024 @@ -1110,11 +1086,10 @@ def filter_reaches(selected_ids): Filter out reaches from selected_ids """ - logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") - if selected_ids is None: subset_reaches_ids = None else: + logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() From eab0fd71baed1718215f8a2afcab286c662f8642 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:12:40 +0100 Subject: [PATCH 051/245] correct else intend --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 8272f6d9..25b8c58a 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -947,7 +947,7 @@ def get_ws_wn_ids(classname): """ if classname is None: ws_wn_ids = None - else: + else: logger.info(f"get list of id's of wastewater_nodes of {classname} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) From 0caa7d3fbc06abc5a8977600f1ece317025f697a Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:17:52 +0100 Subject: [PATCH 052/245] add new selection code to dss export --- qgepqwat2ili/qgepdss/export.py | 66 +++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b2ad7757..8f85be1e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -54,14 +54,64 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # Logging for debugging logger.debug(f"print subset_ids '{subset_ids}'") - # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - - wastewater_structure_id_sia405abwasser_list = None - wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() - - logger.info( - f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", - ) + subset_wws_ids = [] + + flag_approach_urs = True + if flag_approach_urs: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + # 7. If extra_reaches then remove from adapted_subset_ids + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.info( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + # ws_off_sia405abwasser_list = None + # ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + + # 10. Show ws_off_sia405abwasser_list + # logger.info( + # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + #) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + #subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + #logger.info( + # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + #) # Orientation oriented = orientation is not None From 698d2562f226d6def5f341bb3bdc52e3031a98ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:18:07 +0000 Subject: [PATCH 053/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 8f85be1e..802de031 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -7,7 +7,6 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model @@ -106,12 +105,12 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 10. Show ws_off_sia405abwasser_list # logger.info( # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - #) + # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids - #subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) - #logger.info( + # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + # logger.info( # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", - #) + # ) # Orientation oriented = orientation is not None From a12a7231e00d623ebae359e55cebc925023d17e1 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:19:28 +0100 Subject: [PATCH 054/245] add selection code to export kek --- qgepqwat2ili/qgep/export.py | 67 ++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index d49dda50..edc61041 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -43,18 +43,67 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e current_basket = basket_utils.basket_topic_sia405_abwasser - # Filtering + # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] + subset_wws_ids = [] + + flag_approach_urs = True + if flag_approach_urs: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + # 7. If extra_reaches then remove from adapted_subset_ids + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.info( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - - wastewater_structure_id_sia405abwasser_list = None - wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() - - logger.info( - f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", - ) + # 10. Show ws_off_sia405abwasser_list + logger.info( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.info( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) # Orientation oriented = orientation is not None From b2d448a2197bb8c9751128562bd80b37f6e8e137 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:19:53 +0000 Subject: [PATCH 055/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgep/export.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index edc61041..5349ea6a 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -7,7 +7,6 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model @@ -43,7 +42,7 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e current_basket = basket_utils.basket_topic_sia405_abwasser - # 1. Filtering - check if selection + # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] subset_wws_ids = [] From 4767d961e7164cb48867b873359d7c3059ade9e2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:29:37 +0100 Subject: [PATCH 056/245] adapt imports export kek --- qgepqwat2ili/qgep/export.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index edc61041..7b975fb6 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -7,8 +7,19 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids -from ..utils.qgep_export_utils import QgepExportUtils +from ..utils.qgep_export_utils import ( + QgepExportUtils, + add_to_selection, + filter_reaches, + get_connected_overflow_to_wn_ids, + get_connected_we_from_re, + get_connected_we_to_re, + get_ws_ids, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, +) + from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model From 47369713337e9fe6231e90ecbb2c7db9ab778ef2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:30:25 +0100 Subject: [PATCH 057/245] clean up --- qgepqwat2ili/qgepsia405/export.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index fa9c7e22..cc9d4cf1 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -1,14 +1,13 @@ import json from geoalchemy2.functions import ST_Force2D +from sqlalchemy import or_ from sqlalchemy.orm import Session from sqlalchemy.sql import text from .. import utils from ..utils.basket_utils import BasketUtils -# 12.12.2024 - verschoben -# ) from ..utils.qgep_export_utils import ( QgepExportUtils, add_to_selection, @@ -25,17 +24,7 @@ from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model -# 4.10.2024 -# 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added -# from ..utils.ili2db import ( -# add_to_selection, -# filter_reaches, -# get_connected_overflow_to_wn_ids, -# get_connected_we_from_re, -# get_connected_we_to_re, -# get_ws_selected_ww_networkelements, -# get_ws_wn_ids, -# remove_from_selection, + def qgep_export_sia405(selection=None, labels_file=None, orientation=None, basket_enabled=False): From d73bbc0622b5fc6aa096ec726c114b6f7fb98fb9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:32:48 +0000 Subject: [PATCH 058/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgep/export.py | 4 ---- qgepqwat2ili/qgepsia405/export.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index ba13136c..36d92925 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -7,7 +7,6 @@ from .. import utils from ..utils.basket_utils import BasketUtils - from ..utils.qgep_export_utils import ( QgepExportUtils, add_to_selection, @@ -17,11 +16,8 @@ get_connected_we_to_re, get_ws_ids, get_ws_selected_ww_networkelements, - get_ws_wn_ids, remove_from_selection, ) - - from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index cc9d4cf1..e9e84726 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -1,13 +1,11 @@ import json from geoalchemy2.functions import ST_Force2D -from sqlalchemy import or_ from sqlalchemy.orm import Session from sqlalchemy.sql import text from .. import utils from ..utils.basket_utils import BasketUtils - from ..utils.qgep_export_utils import ( QgepExportUtils, add_to_selection, @@ -25,8 +23,6 @@ from .model_qgep import get_qgep_model - - def qgep_export_sia405(selection=None, labels_file=None, orientation=None, basket_enabled=False): """ Export data from the QGEP model into the ili2pg model. From 9be15e6556c9342e5815971000c7922ffcb8e180 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:34:38 +0100 Subject: [PATCH 059/245] adapt import export dss --- qgepqwat2ili/qgepdss/export.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 802de031..07261fa7 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -7,7 +7,18 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.qgep_export_utils import QgepExportUtils +from ..utils.qgep_export_utils import ( + QgepExportUtils, + add_to_selection, + filter_reaches, + get_connected_overflow_to_wn_ids, + get_connected_we_from_re, + get_connected_we_to_re, + get_ws_ids, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, +) from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model @@ -42,7 +53,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e current_basket = basket_utils.basket_topic_sia405_abwasser - # Filtering + # 1. Filtering - check if selection filtered = selection is not None # Logging for debugging From 44028f674a2976611270a887d0c7b5bafae06003 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:36:39 +0000 Subject: [PATCH 060/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 07261fa7..899a9daa 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -14,9 +14,7 @@ get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, - get_ws_ids, get_ws_selected_ww_networkelements, - get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger From ca1b3edb494bc094f985b4e62d0b09b7c37cde2c Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 14:58:12 +0100 Subject: [PATCH 061/245] correct intent of return statement --- qgepqwat2ili/utils/qgep_export_utils.py | 83 +++++++++++++------------ 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 25b8c58a..78141cb6 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -791,7 +791,7 @@ def get_selection_text_for_in_statement(selection_list): selection_text = selection_text[:-1] logger.debug(f"selection_text = {selection_text} ...") - return selection_text + return selection_text # 10.12.2024 @@ -802,45 +802,48 @@ def get_connected_we_from_re(subset_reaches): if subset_reaches is None: connected_wn_from_re_ids = None else: - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_from_re_ids = [] + if not subset_reaches + connected_wn_from_re_ids = None + else: + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + connected_wn_from_re_ids = [] - # select all connected from wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" - ) + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_from_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all connected from wastewater_nodes from provided subset of reaches cursor.execute( f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" ) - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_from_re_ids.append(strrow) - # logger.debug(f" building up '{connected_wn_from_re_ids}' ...") - logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") - return connected_wn_from_re_ids + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_from_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_from_re_ids.append(strrow) + # logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") + return connected_wn_from_re_ids # 10.12.2024 @@ -889,7 +892,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): connected_overflow_to_wn_ids.append(strrow) # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") - return connected_overflow_to_wn_ids + return connected_overflow_to_wn_ids # 10.12.2024 @@ -938,7 +941,7 @@ def get_connected_we_to_re(subset_reaches): connected_wn_to_re_ids.append(strrow) # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") - return connected_wn_to_re_ids + return connected_wn_to_re_ids def get_ws_wn_ids(classname): @@ -982,7 +985,7 @@ def get_ws_wn_ids(classname): ws_wn_ids.append(strrow) # logger.debug(f" building up '{ws_wn_ids}' ...") - return ws_wn_ids + return ws_wn_ids # 12.12.2024 @@ -1024,7 +1027,7 @@ def get_ws_ids(classname): ws_ids.append(strrow) # logger.debug(f" building up '{ws_wn_ids}' ...") - return ws_ids + return ws_ids def get_ws_selected_ww_networkelements(selected_wwn): @@ -1077,7 +1080,7 @@ def get_ws_selected_ww_networkelements(selected_wwn): ws_ids.append(strrow) # logger.debug(f" building up '{ws_wn_ids}' ...") - return ws_ids + return ws_ids # 10.1.2024 @@ -1151,7 +1154,7 @@ def remove_from_selection(selected_ids, remove_ids): f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" ) - return selected_ids + return selected_ids def add_to_selection(selected_ids, add_ids): @@ -1168,4 +1171,4 @@ def add_to_selection(selected_ids, add_ids): # selected_ids = selected_ids.append(list_item) selected_ids.append(list_item) - return selected_ids + return selected_ids From fd8de5880fbaad62d1136cd3199d8f6083432a01 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 15:00:40 +0100 Subject: [PATCH 062/245] add : and comment --- qgepqwat2ili/utils/qgep_export_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 78141cb6..902902a0 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -802,7 +802,8 @@ def get_connected_we_from_re(subset_reaches): if subset_reaches is None: connected_wn_from_re_ids = None else: - if not subset_reaches + # check if list is empty https://stackoverflow.com/questions/53513/how-do-i-check-if-a-list-is-empty + if not subset_reaches: connected_wn_from_re_ids = None else: logger.info( From 8eb5752f52dd4e4ba081f84347e523be54a035a2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 15:10:12 +0100 Subject: [PATCH 063/245] def get_selection_text_for_in_statement check if list is empty --- qgepqwat2ili/qgepsia405/export.py | 1 + qgepqwat2ili/utils/qgep_export_utils.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index e9e84726..9c0b09bb 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -66,6 +66,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 902902a0..474baa14 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -780,17 +780,21 @@ def get_selection_text_for_in_statement(selection_list): if selection_list is None: selection_text = None else: - selection_text = "" + if not selection_list: + # list is empty - no need for adaption + selection_text = None + else: + selection_text = "" - for list_item in selection_list: - selection_text += "'" - selection_text += list_item - selection_text += "'," + for list_item in selection_list: + selection_text += "'" + selection_text += list_item + selection_text += "'," - # remove last komma to make it a correct IN statement - selection_text = selection_text[:-1] + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] - logger.debug(f"selection_text = {selection_text} ...") + logger.debug(f"selection_text = {selection_text} ...") return selection_text From 2d71f2bb05467a09b133a033d871d048de931e96 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 15:19:11 +0100 Subject: [PATCH 064/245] check for empty list added --- qgepqwat2ili/utils/qgep_export_utils.py | 155 ++++++++++++------------ 1 file changed, 79 insertions(+), 76 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 474baa14..10e86ab5 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -852,101 +852,104 @@ def get_connected_we_from_re(subset_reaches): # 10.12.2024 -def get_connected_overflow_to_wn_ids(selected_ids): +def get_connected_we_to_re(subset_reaches): """ - Get all connected wastewater_nodes from overflows.fk_overflow_to + Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches """ - if selected_ids is None: - connected_overflow_to_wn_ids = None - else: - logger.info( - f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_overflow_to_wn_ids = [] + if subset_reaches is None: + connected_wn_to_re_ids = None + if not subset_reaches: + connected_wn_to_re_ids = None + else: + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - subset_text = get_selection_text_for_in_statement(selected_ids) + connected_wn_to_re_ids = [] - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" - ) + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_overflow_to_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" + f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" ) - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_overflow_to_wn_ids.append(strrow) - # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") - logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") - return connected_overflow_to_wn_ids + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" + ) + records = cursor.fetchall() + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") + return connected_wn_to_re_ids # 10.12.2024 -def get_connected_we_to_re(subset_reaches): +def get_connected_overflow_to_wn_ids(selected_ids): """ - Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches + Get all connected wastewater_nodes from overflows.fk_overflow_to """ - if subset_reaches is None: - connected_wn_to_re_ids = None - else: - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() - - connected_wn_to_re_ids = [] + if selected_ids is None: + connected_overflow_to_wn_ids = None + if not selected_ids: + connected_overflow_to_wn_ids = None + else: + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + connected_overflow_to_wn_ids = [] - # select all connected to wastewater_nodes from provided subset of reaches - cursor.execute( - f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" - ) + subset_text = get_selection_text_for_in_statement(selected_ids) - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_to_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( - f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" ) - records = cursor.fetchall() - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_to_re_ids.append(strrow) - # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") - return connected_wn_to_re_ids + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") + return connected_overflow_to_wn_ids def get_ws_wn_ids(classname): From 00852a117b1501e077296f7356eafeaf12f91a4e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:19:26 +0000 Subject: [PATCH 065/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 10e86ab5..845ac867 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -901,6 +901,7 @@ def get_connected_we_to_re(subset_reaches): logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids + # 10.12.2024 def get_connected_overflow_to_wn_ids(selected_ids): """ From 47255baebe74c2ad8765564d9dee5ae8f30b2425 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 15:33:47 +0100 Subject: [PATCH 066/245] adapt case None --- qgepqwat2ili/utils/qgep_export_utils.py | 152 ++++++++++++------------ 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 10e86ab5..0ac563f2 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -857,48 +857,48 @@ def get_connected_we_to_re(subset_reaches): Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches """ if subset_reaches is None: - connected_wn_to_re_ids = None - if not subset_reaches: - connected_wn_to_re_ids = None - else: - logger.info( - f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() + return None + if not subset_reaches: + return None + else: + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() - connected_wn_to_re_ids = [] + connected_wn_to_re_ids = [] - subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" + ) - # select all connected to wastewater_nodes from provided subset of reaches + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working cursor.execute( f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" ) + records = cursor.fetchall() - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_wn_to_re_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT wet.obj_id as wet_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpt ON rpt.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wet ON wet.obj_id = rpt.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wet.obj_id isNull;" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_wn_to_re_ids.append(strrow) - # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids # 10.12.2024 @@ -907,47 +907,47 @@ def get_connected_overflow_to_wn_ids(selected_ids): Get all connected wastewater_nodes from overflows.fk_overflow_to """ if selected_ids is None: - connected_overflow_to_wn_ids = None - if not selected_ids: - connected_overflow_to_wn_ids = None - else: - logger.info( - f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." - ) - connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) - connection.set_session(autocommit=True) - cursor = connection.cursor() + return None + if not selected_ids: + return None + else: + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_overflow_to_wn_ids = [] - connected_overflow_to_wn_ids = [] + subset_text = get_selection_text_for_in_statement(selected_ids) - subset_text = get_selection_text_for_in_statement(selected_ids) + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" + ) - # select all connected to wastewater_nodes from provided subset of reaches + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working cursor.execute( f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" ) + records = cursor.fetchall() - # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - # ws_wn_ids_count = int(cursor.fetchone()[0]) - # if ws_wn_ids_count == 0: - if cursor.fetchone() is None: - connected_overflow_to_wn_ids = None - else: - # added cursor.execute again to see if with this all records will be available - # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working - cursor.execute( - f"SELECT ov.fk_overflow_to FROM qgep_od.wastewater_node wn LEFT JOIN qgep_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text}) AND NOT ov.fk_overflow_to isNULL;" - ) - records = cursor.fetchall() - - # 15.11.2024 - does not get all records, but only n-1 - for row in records: - logger.debug(f" row[0] = {row[0]}") - # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ - strrow = str(row[0]) - if strrow is not None: - connected_overflow_to_wn_ids.append(strrow) - # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") return connected_overflow_to_wn_ids @@ -957,7 +957,7 @@ def get_ws_wn_ids(classname): Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) """ if classname is None: - ws_wn_ids = None + return None else: logger.info(f"get list of id's of wastewater_nodes of {classname} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) @@ -1003,7 +1003,7 @@ def get_ws_ids(classname): """ if classname is None: - ws_ids = None + return None else: logger.info(f"get list of id's of subclass {classname} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) @@ -1044,7 +1044,7 @@ def get_ws_selected_ww_networkelements(selected_wwn): """ if selected_wwn is None: - ws_ids = None + return None else: logger.debug( @@ -1098,7 +1098,7 @@ def filter_reaches(selected_ids): """ if selected_ids is None: - subset_reaches_ids = None + return None else: logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) @@ -1141,7 +1141,7 @@ def filter_reaches(selected_ids): ) else: logger.debug(f"'filter_reaches: {list_item}' is not a reach id") - logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") + logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") return subset_reaches_ids @@ -1151,7 +1151,7 @@ def remove_from_selection(selected_ids, remove_ids): """ if remove_ids is None: - return selected_ids + return None else: for list_item in remove_ids: # selected_ids = selected_ids.remove(list_item) @@ -1170,7 +1170,7 @@ def add_to_selection(selected_ids, add_ids): Append ids to selected_ids """ if add_ids is None: - return add_to_selection + return None else: if selected_ids is None: selected_ids = [] From 20cbde0693e8cfed330fa1861bdac6789a0a8af6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 15:39:48 +0100 Subject: [PATCH 067/245] check for empty adapted_subset_ids_reaches --- qgepqwat2ili/qgepsia405/export.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9c0b09bb..09585d47 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -73,17 +73,22 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + if not adapted_subset_ids_reaches: + logger.debug( + "no adapted_subset_ids_reaches - so nothing to remove", + ) + else: + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) # 7. If extra_reaches then remove from adapted_subset_ids if not extra_reaches_ids: # list is empty - no need for adaption From aacaad25c91ccb45d8194ca0a574626591492ccb Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 15:45:20 +0100 Subject: [PATCH 068/245] check for None --- qgepqwat2ili/qgepsia405/export.py | 57 ++++++++++++++++--------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 09585d47..8bfeef9e 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -73,34 +73,35 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - if not adapted_subset_ids_reaches: - logger.debug( - "no adapted_subset_ids_reaches - so nothing to remove", - ) - else: - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) - # 7. If extra_reaches then remove from adapted_subset_ids - if not extra_reaches_ids: - # list is empty - no need for adaption - logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", - ) - else: - logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", - ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + if adapted_subset_ids_reaches is None: + if not adapted_subset_ids_reaches: + logger.debug( + "no adapted_subset_ids_reaches - so nothing to remove", + ) + else: + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + # 7. If extra_reaches then remove from adapted_subset_ids + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( From e56fa8b0da0b3286be00959582a96de398d0b6fd Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 16:09:31 +0100 Subject: [PATCH 069/245] add debug.logger --- qgepqwat2ili/qgepsia405/export.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 8bfeef9e..8c8acb5e 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -79,6 +79,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske "no adapted_subset_ids_reaches - so nothing to remove", ) else: + logger.debug( + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) # https://www.geeksforgeeks.org/python-difference-two-lists/ # First convert lists to sets # https://www.w3schools.com/python/ref_set_difference.asp From 20e02786d663ec66fa205b60956c0b2a185d9ce2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 16:16:49 +0100 Subject: [PATCH 070/245] check none --- qgepqwat2ili/qgepsia405/export.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 8c8acb5e..1f4ef1ac 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -94,17 +94,18 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # c = list(set(a) - set(b)) extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) # 7. If extra_reaches then remove from adapted_subset_ids - if not extra_reaches_ids: - # list is empty - no need for adaption - logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", - ) - else: - logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", - ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( From e4809e4baeb15bd4ca23f9d40e297c5feb26b71b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:17:12 +0000 Subject: [PATCH 071/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 1f4ef1ac..48845fad 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -105,7 +105,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske f"extra_reaches_ids: {extra_reaches_ids} found!", ) # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids + ) # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( From c047de2eae76f1896d7487d835c489f68e1623c4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 16:22:48 +0100 Subject: [PATCH 072/245] initialize extra_reaches_ids --- qgepqwat2ili/qgepsia405/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 1f4ef1ac..e980d7d3 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -74,6 +74,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] if not adapted_subset_ids_reaches: logger.debug( "no adapted_subset_ids_reaches - so nothing to remove", From 462b60db952648572389a52258c0eb11a8072a12 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 16:43:03 +0100 Subject: [PATCH 073/245] if no selection subset will be None --- qgepqwat2ili/utils/qgep_export_utils.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index f44ff712..560079d3 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -115,18 +115,22 @@ def check_fk_in_subsetid(self, subset, relation): fremdschluesselstr = getattr(relation, "obj_id") logger.info(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") - if fremdschluesselstr in subset: - logger.info(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") - logger.info(f"check_fk_in_subsetid - tid = '{self.tid_maker.tid_for_row(relation)}' ") + # if no selection subset will be None + if subset is None: return self.tid_maker.tid_for_row(relation) else: - if self.filtered: - logger.warning( - f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" - ) - return None - else: + if fremdschluesselstr in subset: + logger.info(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") + logger.info(f"check_fk_in_subsetid - tid = '{self.tid_maker.tid_for_row(relation)}' ") return self.tid_maker.tid_for_row(relation) + else: + if self.filtered: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" + ) + return None + else: + return self.tid_maker.tid_for_row(relation) def create_metaattributes(self, row): metaattribute = self.abwasser_model.metaattribute( From 2cdb4c1d5bad2a3d992e1ff5856037a496bef0a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:43:22 +0000 Subject: [PATCH 074/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 560079d3..28508e2f 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -121,7 +121,9 @@ def check_fk_in_subsetid(self, subset, relation): else: if fremdschluesselstr in subset: logger.info(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") - logger.info(f"check_fk_in_subsetid - tid = '{self.tid_maker.tid_for_row(relation)}' ") + logger.info( + f"check_fk_in_subsetid - tid = '{self.tid_maker.tid_for_row(relation)}' " + ) return self.tid_maker.tid_for_row(relation) else: if self.filtered: From faa5a95efb57e31e0d3195ea1a584bc4315e2726 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 16:50:24 +0100 Subject: [PATCH 075/245] update qgep export dss and kek --- qgepqwat2ili/qgep/export.py | 61 +++++++++++++++++++------------ qgepqwat2ili/qgepdss/export.py | 66 +++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 48 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 36d92925..ba7a8bd0 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -16,6 +16,7 @@ get_connected_we_to_re, get_ws_ids, get_ws_selected_ww_networkelements, + get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger @@ -66,35 +67,49 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) - # 7. If extra_reaches then remove from adapted_subset_ids - if not extra_reaches_ids: - # list is empty - no need for adaption - logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", - ) - else: - logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", - ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] + if not adapted_subset_ids_reaches: + logger.debug( + "no adapted_subset_ids_reaches - so nothing to remove", + ) + else: + logger.debug( + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + # 7. If extra_reaches then remove from adapted_subset_ids + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids + ) # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 899a9daa..b36a2026 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -14,7 +14,9 @@ get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, + get_ws_ids, get_ws_selected_ww_networkelements, + get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger @@ -73,35 +75,49 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) - # 7. If extra_reaches then remove from adapted_subset_ids - if not extra_reaches_ids: - # list is empty - no need for adaption - logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", - ) - else: - logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", - ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] + if not adapted_subset_ids_reaches: + logger.debug( + "no adapted_subset_ids_reaches - so nothing to remove", + ) + else: + logger.debug( + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + # 7. If extra_reaches then remove from adapted_subset_ids + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids + ) # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( @@ -113,12 +129,12 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 10. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", # ) # Orientation From 9f9f9efda11aa7a5a0d7a2d581fd5318a86fea1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:52:42 +0000 Subject: [PATCH 076/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgep/export.py | 1 - qgepqwat2ili/qgepdss/export.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index ba7a8bd0..487ced4a 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -16,7 +16,6 @@ get_connected_we_to_re, get_ws_ids, get_ws_selected_ww_networkelements, - get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b36a2026..fe788fec 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -14,9 +14,7 @@ get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, - get_ws_ids, get_ws_selected_ww_networkelements, - get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger @@ -129,12 +127,12 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 10. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", # ) # Orientation From d9244c1ba31be629d0f311cc520ca10673b656b7 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 17:07:09 +0100 Subject: [PATCH 077/245] only if filtered --- qgepqwat2ili/qgepsia405/export.py | 217 +++++++++++++++--------------- 1 file changed, 109 insertions(+), 108 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 3b11ab4e..20184655 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -57,120 +57,121 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske subset_ids = selection if selection is not None else [] subset_wws_ids = [] - flag_approach_urs = True - if flag_approach_urs: - # 2. Get all connected from wastewater_nodes of selected reaches - connected_from_wn_ids = get_connected_we_from_re(subset_ids) - # 3. Get all connected to wastewater_nodes of selected reaches - connected_to_wn_ids = get_connected_we_to_re(subset_ids) - # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) - # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids - adapted_subset_ids = [] - adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) - # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids - subset_ids_reaches = filter_reaches(subset_ids) - adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - if adapted_subset_ids_reaches is None: - extra_reaches_ids = [] - if not adapted_subset_ids_reaches: - logger.debug( - "no adapted_subset_ids_reaches - so nothing to remove", - ) - else: - logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", - ) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) - # 7. If extra_reaches then remove from adapted_subset_ids - if extra_reaches_ids is None: - if not extra_reaches_ids: - # list is empty - no need for adaption + if filtered: + flag_approach_urs = True + if flag_approach_urs: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] + if not adapted_subset_ids_reaches: logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", + "no adapted_subset_ids_reaches - so nothing to remove", ) else: logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection( - adapted_subset_ids, extra_reaches_ids + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + # 7. If extra_reaches then remove from adapted_subset_ids + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids + ) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.info( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + + # 10. Show ws_off_sia405abwasser_list + logger.info( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.info( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) + else: # flag_approach_urs = False + # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + + # 3. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + + # 4. check if filtered + if filtered: + if ws_off_sia405abwasser_list: + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + # else do nothing + else: + if ws_off_sia405abwasser_list: + # add all data of wastewater_structures to selection + subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) + logger.debug( + f"subset_ids of all wws : {subset_ids}", ) - # 8. get all id's of connected wastewater_structures - subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) - logger.info( - f"subset_wws_ids: {subset_wws_ids}", - ) - # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) - ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - - # 10. Show ws_off_sia405abwasser_list - logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - ) - # 11. take out ws_off_sia405abwasser_list from subset_wws_ids - subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) - logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", - ) - else: # flag_approach_urs = False - # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) - ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") - - # 3. Show ws_off_sia405abwasser_list - logger.debug( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - ) - - # 4. check if filtered - if filtered: - if ws_off_sia405abwasser_list: - # take out ws_off_sia405abwasser_list from selection - subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) - # else do nothing - else: - if ws_off_sia405abwasser_list: - # add all data of wastewater_structures to selection - subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) - logger.debug( - f"subset_ids of all wws : {subset_ids}", - ) - # take out ws_off_sia405abwasser_list from selection - subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) - logger.debug( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", - ) - # add reach_ids - # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) - # treat export as with a selection - filtered = True - - # else do nothing - - # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) - subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) - logger.debug( - f"subset_wws_ids: {subset_wws_ids}", - ) - subset_ids = add_to_selection(subset_ids, subset_wws_ids) - logger.debug( - f"subset_ids with wws : {subset_ids}", - ) + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + ) + # add reach_ids + # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) + # treat export as with a selection + filtered = True + + # else do nothing + + # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) + subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) + logger.debug( + f"subset_wws_ids: {subset_wws_ids}", + ) + subset_ids = add_to_selection(subset_ids, subset_wws_ids) + logger.debug( + f"subset_ids with wws : {subset_ids}", + ) # Orientation oriented = orientation is not None From 14cc560b7ba52b5ddf67b83b73309e1baefc79a4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:07:35 +0000 Subject: [PATCH 078/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 20184655..51e7dda4 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -94,7 +94,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + extra_reaches_ids = list( + set(subset_ids_reaches) - set(adapted_subset_ids_reaches) + ) # 7. If extra_reaches then remove from adapted_subset_ids if extra_reaches_ids is None: if not extra_reaches_ids: @@ -147,7 +149,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske else: if ws_off_sia405abwasser_list: # add all data of wastewater_structures to selection - subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) + subset_ids = add_to_selection( + subset_ids, get_ws_wn_ids("wastewater_structure") + ) logger.debug( f"subset_ids of all wws : {subset_ids}", ) From dad451821e230de7925b82a32c435b151ae24ffd Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 17:23:26 +0100 Subject: [PATCH 079/245] add wwtp_structure case also to not filtered --- qgepqwat2ili/qgepsia405/export.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 20184655..1efe060e 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -173,6 +173,25 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske f"subset_ids with wws : {subset_ids}", ) + # also if not filtered we have to take out references to wwtp_structures + else: + # 20. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + + # 21. Show ws_off_sia405abwasser_list + logger.info( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + + # 22. Get list of all wastewater_structures + subset_wws_ids = get_ws_ids("wastewater_structure") + # 23. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.info( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) + # Orientation oriented = orientation is not None if oriented: From 32be05d933bfa036431b1c7127447a00ae596b30 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:26:03 +0000 Subject: [PATCH 080/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 06d04547..fedbc0f0 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -187,7 +187,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info( f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", ) - + # 22. Get list of all wastewater_structures subset_wws_ids = get_ws_ids("wastewater_structure") # 23. take out ws_off_sia405abwasser_list from subset_wws_ids From 09d25c3d4ed049884d24b4df2ffe91d2f2356d58 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 12 Dec 2024 17:29:33 +0100 Subject: [PATCH 081/245] add logger infos --- qgepqwat2ili/qgepsia405/export.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 06d04547..0fd4cbae 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -185,15 +185,18 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 21. Show ws_off_sia405abwasser_list logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", ) # 22. Get list of all wastewater_structures subset_wws_ids = get_ws_ids("wastewater_structure") + logger.info( + f"subset_wws_ids (non filtered) : {subset_wws_ids}", + ) # 23. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", ) # Orientation From a455fd60f9156878f9c7cb842e5388a88ebc44cd Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 14:43:37 +0100 Subject: [PATCH 082/245] #if self.filtered: --- qgepqwat2ili/utils/qgep_export_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 28508e2f..ab529254 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -126,13 +126,14 @@ def check_fk_in_subsetid(self, subset, relation): ) return self.tid_maker.tid_for_row(relation) else: - if self.filtered: - logger.warning( - f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" - ) - return None - else: - return self.tid_maker.tid_for_row(relation) + # take out - as it has to work also without filtered (SIA405 Abwasser) + #if self.filtered: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" + ) + return None + #else: + # return self.tid_maker.tid_for_row(relation) def create_metaattributes(self, row): metaattribute = self.abwasser_model.metaattribute( From 4f03d77089432bff18396dc34ee1ab8289dfe3c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:43:53 +0000 Subject: [PATCH 083/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index ab529254..e2f0d653 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -127,12 +127,12 @@ def check_fk_in_subsetid(self, subset, relation): return self.tid_maker.tid_for_row(relation) else: # take out - as it has to work also without filtered (SIA405 Abwasser) - #if self.filtered: + # if self.filtered: logger.warning( f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" ) return None - #else: + # else: # return self.tid_maker.tid_for_row(relation) def create_metaattributes(self, row): From 749bafc6565e1f8ff519e7768bf17cea3793d83c Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 14:52:25 +0100 Subject: [PATCH 084/245] change to info and adapt message --- qgepqwat2ili/qgepsia405/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 966db784..05735ece 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -299,8 +299,8 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # --- _rel_ --- # accessibility__REL, defects__REL, emergency_spillway__REL, financing__REL, fk_aquifier__REL, fk_dataowner__REL, fk_main_cover__REL, fk_main_wastewater_node__REL, fk_operator__REL, fk_owner__REL, fk_provider__REL, kind__REL, labeling__REL, renovation_necessity__REL, rv_construction_type__REL, seepage_utilization__REL, status__REL, structure_condition__REL, vehicle_access__REL, watertightness__REL - logger.warning( - "QGEP field infiltration_installation.upper_elevation has no equivalent in the interlis model. It will be ignored." + logger.info( + "QGEP field infiltration_installation.upper_elevation is part of 3D extension. It will be ignored." ) versickerungsanlage = abwasser_model.versickerungsanlage( # FIELDS TO MAP TO ABWASSER.versickerungsanlage From 2e1685317ec87c1c6d0cd446970a4f6048d9af8a Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 15:01:44 +0100 Subject: [PATCH 085/245] adapt logger levels --- qgepqwat2ili/utils/qgep_export_utils.py | 60 ++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index e2f0d653..b4c3d829 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -110,18 +110,18 @@ def check_fk_in_subsetid(self, subset, relation): if relation is None: return None - logger.info(f"check_fk_in_subsetid - Subset ID's '{subset}'") + logger.debug(f"check_fk_in_subsetid - Subset ID's '{subset}'") # get the value of the fk_ attribute as str out of the relation to be able to check whether it is in the subset fremdschluesselstr = getattr(relation, "obj_id") - logger.info(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") + logger.debug(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") # if no selection subset will be None if subset is None: return self.tid_maker.tid_for_row(relation) else: if fremdschluesselstr in subset: - logger.info(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") - logger.info( + logger.debug(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") + logger.debug( f"check_fk_in_subsetid - tid = '{self.tid_maker.tid_for_row(relation)}' " ) return self.tid_maker.tid_for_row(relation) @@ -184,9 +184,9 @@ def wastewater_structure_common(self, row): """ Returns common attributes for wastewater_structure """ - logger.warning( - "Mapping of wastewater_structure->abwasserbauwerk is not fully implemented." - ) + # logger.warning( + # "Mapping of wastewater_structure->abwasserbauwerk is not yet implemented for 3D extensions of SIA405 Abwasser, VSA-KEK and VSA-DSS 2015." + # ) return { # --- abwasserbauwerk --- "akten": row.records, @@ -817,7 +817,7 @@ def get_connected_we_from_re(subset_reaches): if not subset_reaches: connected_wn_from_re_ids = None else: - logger.info( + logger.debug( f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." ) connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) @@ -848,13 +848,13 @@ def get_connected_we_from_re(subset_reaches): # 15.11.2024 - does not get all records, but only n-1 for row in records: - logger.debug(f" row[0] = {row[0]}") + # logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: connected_wn_from_re_ids.append(strrow) # logger.debug(f" building up '{connected_wn_from_re_ids}' ...") - logger.info(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") + logger.debug(f" connected_wn_from_re_ids: '{connected_wn_from_re_ids}'") return connected_wn_from_re_ids @@ -868,7 +868,7 @@ def get_connected_we_to_re(subset_reaches): if not subset_reaches: return None else: - logger.info( + logger.debug( f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." ) connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) @@ -899,13 +899,13 @@ def get_connected_we_to_re(subset_reaches): # 15.11.2024 - does not get all records, but only n-1 for row in records: - logger.debug(f" row[0] = {row[0]}") + # logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: connected_wn_to_re_ids.append(strrow) # logger.debug(f" building up '{connected_wn_to_re_ids}' ...") - logger.info(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") + logger.debug(f" connected_wn_to_re_ids: '{connected_wn_to_re_ids}'") return connected_wn_to_re_ids @@ -919,7 +919,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): if not selected_ids: return None else: - logger.info( + logger.debug( f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." ) connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) @@ -950,13 +950,13 @@ def get_connected_overflow_to_wn_ids(selected_ids): # 15.11.2024 - does not get all records, but only n-1 for row in records: - logger.debug(f" row[0] = {row[0]}") + # logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: connected_overflow_to_wn_ids.append(strrow) # logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") - logger.info(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") + logger.debug(f" connected_overflow_to_wn_ids: '{connected_overflow_to_wn_ids}'") return connected_overflow_to_wn_ids @@ -967,7 +967,7 @@ def get_ws_wn_ids(classname): if classname is None: return None else: - logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + logger.debug(f"get list of id's of wastewater_nodes of {classname} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() @@ -994,7 +994,7 @@ def get_ws_wn_ids(classname): # 15.11.2024 - does not get all records, but only n-1 for row in records: - logger.debug(f" row[0] = {row[0]}") + # logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: @@ -1013,7 +1013,7 @@ def get_ws_ids(classname): if classname is None: return None else: - logger.info(f"get list of id's of subclass {classname} ...") + logger.debug(f"get list of id's of subclass {classname} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() @@ -1036,13 +1036,13 @@ def get_ws_ids(classname): # 15.11.2024 - does not get all records, but only n-1 for row in records: - logger.debug(f" row[0] = {row[0]}") + # logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: ws_ids.append(strrow) # logger.debug(f" building up '{ws_wn_ids}' ...") - + logger.debug(f" ws_ids: '{ws_ids}' ...") return ws_ids @@ -1074,7 +1074,7 @@ def get_ws_selected_ww_networkelements(selected_wwn): logger.debug(f"selection_text = {selection_text} ...") - ws_ids = [] + ws_selected_ww_networkelements_ids = [] # select all obj_id of the wastewater_nodes of wwtp_structure cursor.execute( @@ -1085,7 +1085,7 @@ def get_ws_selected_ww_networkelements(selected_wwn): # ws_wn_ids_count = int(cursor.fetchone()[0]) # if ws_wn_ids_count == 0: if cursor.fetchone() is None: - ws_ids = None + ws_selected_ww_networkelements_ids = None else: records = cursor.fetchall() for row in records: @@ -1093,10 +1093,10 @@ def get_ws_selected_ww_networkelements(selected_wwn): # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: - ws_ids.append(strrow) - # logger.debug(f" building up '{ws_wn_ids}' ...") - - return ws_ids + ws_selected_ww_networkelements_ids.append(strrow) + # logger.debug(f" building up '{ws_selected_ww_networkelements_ids}' ...") + logger.debug(f" ws_selected_ww_networkelements_ids: '{ws_selected_ww_networkelements_ids}' ...") + return ws_selected_ww_networkelements_ids # 10.1.2024 @@ -1108,7 +1108,7 @@ def filter_reaches(selected_ids): if selected_ids is None: return None else: - logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") + logger.debug(f"Filter out reaches from selected_ids {selected_ids} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() @@ -1134,7 +1134,7 @@ def filter_reaches(selected_ids): # 15.11.2024 - does not get all records, but only n-1 for row in records: - logger.debug(f" row[0] = {row[0]}") + # logger.debug(f" row[0] = {row[0]}") # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ strrow = str(row[0]) if strrow is not None: @@ -1149,7 +1149,7 @@ def filter_reaches(selected_ids): ) else: logger.debug(f"'filter_reaches: {list_item}' is not a reach id") - logger.info(f"'subset_reaches_ids: {subset_reaches_ids}'") + logger.debug(f"'subset_reaches_ids: {subset_reaches_ids}'") return subset_reaches_ids From 34c24b66fb29450c41d366ad280b0fff8dbc8c6d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:03:12 +0000 Subject: [PATCH 086/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index b4c3d829..9284c60d 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -185,7 +185,7 @@ def wastewater_structure_common(self, row): Returns common attributes for wastewater_structure """ # logger.warning( - # "Mapping of wastewater_structure->abwasserbauwerk is not yet implemented for 3D extensions of SIA405 Abwasser, VSA-KEK and VSA-DSS 2015." + # "Mapping of wastewater_structure->abwasserbauwerk is not yet implemented for 3D extensions of SIA405 Abwasser, VSA-KEK and VSA-DSS 2015." # ) return { # --- abwasserbauwerk --- @@ -1095,7 +1095,9 @@ def get_ws_selected_ww_networkelements(selected_wwn): if strrow is not None: ws_selected_ww_networkelements_ids.append(strrow) # logger.debug(f" building up '{ws_selected_ww_networkelements_ids}' ...") - logger.debug(f" ws_selected_ww_networkelements_ids: '{ws_selected_ww_networkelements_ids}' ...") + logger.debug( + f" ws_selected_ww_networkelements_ids: '{ws_selected_ww_networkelements_ids}' ..." + ) return ws_selected_ww_networkelements_ids From c853a493e90ea505957bce7f1363322636b01d14 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 15:09:03 +0100 Subject: [PATCH 087/245] update export kek with latest changes from sia405 --- qgepqwat2ili/qgep/export.py | 187 +++++++++++++++++++++++++----------- 1 file changed, 129 insertions(+), 58 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 487ced4a..4642193a 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -57,75 +57,146 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e subset_ids = selection if selection is not None else [] subset_wws_ids = [] - flag_approach_urs = True - if flag_approach_urs: - # 2. Get all connected from wastewater_nodes of selected reaches - connected_from_wn_ids = get_connected_we_from_re(subset_ids) - # 3. Get all connected to wastewater_nodes of selected reaches - connected_to_wn_ids = get_connected_we_to_re(subset_ids) - # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) - # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids - adapted_subset_ids = [] - adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) - # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids - subset_ids_reaches = filter_reaches(subset_ids) - adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - if adapted_subset_ids_reaches is None: - extra_reaches_ids = [] - if not adapted_subset_ids_reaches: - logger.debug( - "no adapted_subset_ids_reaches - so nothing to remove", - ) - else: - logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", - ) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) - # 7. If extra_reaches then remove from adapted_subset_ids - if extra_reaches_ids is None: - if not extra_reaches_ids: - # list is empty - no need for adaption + if filtered: + flag_approach_urs = True + if flag_approach_urs: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] + if not adapted_subset_ids_reaches: logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", + "no adapted_subset_ids_reaches - so nothing to remove", ) else: logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection( - adapted_subset_ids, extra_reaches_ids + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list( + set(subset_ids_reaches) - set(adapted_subset_ids_reaches) ) - # 8. get all id's of connected wastewater_structures - subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) - logger.info( - f"subset_wws_ids: {subset_wws_ids}", - ) - # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + # 7. If extra_reaches then remove from adapted_subset_ids + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + logger.debug( + f"extra_reaches_ids: {extra_reaches_ids} found!", + ) + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids + ) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.info( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + + # 10. Show ws_off_sia405abwasser_list + logger.info( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.info( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) + else: # flag_approach_urs = False + # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + + # 3. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + + # 4. check if filtered + if filtered: + if ws_off_sia405abwasser_list: + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + # else do nothing + else: + if ws_off_sia405abwasser_list: + # add all data of wastewater_structures to selection + subset_ids = add_to_selection( + subset_ids, get_ws_wn_ids("wastewater_structure") + ) + logger.debug( + f"subset_ids of all wws : {subset_ids}", + ) + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + ) + # add reach_ids + # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) + # treat export as with a selection + filtered = True + + # else do nothing + + # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) + subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) + logger.debug( + f"subset_wws_ids: {subset_wws_ids}", + ) + subset_ids = add_to_selection(subset_ids, subset_wws_ids) + logger.debug( + f"subset_ids with wws : {subset_ids}", + ) + + # also if not filtered we have to take out references to wwtp_structures + else: + # 20. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) ws_off_sia405abwasser_list = None ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - # 10. Show ws_off_sia405abwasser_list + # 21. Show ws_off_sia405abwasser_list logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", ) - # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + + # 22. Get list of all wastewater_structures + subset_wws_ids = get_ws_ids("wastewater_structure") + logger.info( + f"subset_wws_ids (non filtered) : {subset_wws_ids}", + ) + # 23. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", ) # Orientation @@ -228,8 +299,8 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e # --- _rel_ --- # accessibility__REL, defects__REL, emergency_spillway__REL, financing__REL, fk_aquifier__REL, fk_dataowner__REL, fk_main_cover__REL, fk_main_wastewater_node__REL, fk_operator__REL, fk_owner__REL, fk_provider__REL, kind__REL, labeling__REL, renovation_necessity__REL, rv_construction_type__REL, seepage_utilization__REL, status__REL, structure_condition__REL, vehicle_access__REL, watertightness__REL - logger.warning( - "QGEP field infiltration_installation.upper_elevation has no equivalent in the interlis model. It will be ignored." + logger.info( + "QGEP field infiltration_installation.upper_elevation is part of 3D extension. It will be ignored." ) versickerungsanlage = abwasser_model.versickerungsanlage( # FIELDS TO MAP TO ABWASSER.versickerungsanlage From 00cae22b7e66121790a5e4ef41adf0adaa357a68 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 15:17:10 +0100 Subject: [PATCH 088/245] update export dss with changes from sia405 abwasser --- qgepqwat2ili/qgepdss/export.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index fe788fec..dfafa994 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -64,8 +64,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e subset_wws_ids = [] - flag_approach_urs = True - if flag_approach_urs: + if filtered: # 2. Get all connected from wastewater_nodes of selected reaches connected_from_wn_ids = get_connected_we_from_re(subset_ids) # 3. Get all connected to wastewater_nodes of selected reaches @@ -100,7 +99,9 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) - extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) + extra_reaches_ids = list( + set(subset_ids_reaches) - set(adapted_subset_ids_reaches) + ) # 7. If extra_reaches then remove from adapted_subset_ids if extra_reaches_ids is None: if not extra_reaches_ids: @@ -127,13 +128,36 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 10. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # ) + + # also if not filtered we have to take out references to wwtp_structures + else: + # 20. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + # ws_off_sia405abwasser_list = None + # ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + + # 21. Show ws_off_sia405abwasser_list + # logger.info( + # f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", + # ) + + # 22. Get list of all wastewater_structures + # subset_wws_ids = get_ws_ids("wastewater_structure") + # logger.info( + # f"subset_wws_ids (non filtered) : {subset_wws_ids}", + # ) + # 23. take out ws_off_sia405abwasser_list from subset_wws_ids + # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + # logger.info( + # f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", # ) + logger.debug("Handling of wwtp_structures not needed with VSA-DSS") # Orientation oriented = orientation is not None From c021f20f28f0fa9b5f1231117d68235342f532cd Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 15:17:48 +0100 Subject: [PATCH 089/245] take out old approach - keep only approach_urs - no if needed anymore --- qgepqwat2ili/qgepsia405/export.py | 176 +++++++++++------------------- 1 file changed, 64 insertions(+), 112 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 05735ece..65722ad0 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -58,124 +58,76 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske subset_wws_ids = [] if filtered: - flag_approach_urs = True - if flag_approach_urs: - # 2. Get all connected from wastewater_nodes of selected reaches - connected_from_wn_ids = get_connected_we_from_re(subset_ids) - # 3. Get all connected to wastewater_nodes of selected reaches - connected_to_wn_ids = get_connected_we_to_re(subset_ids) - # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) - # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids - adapted_subset_ids = [] - adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) - # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids - subset_ids_reaches = filter_reaches(subset_ids) - adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - if adapted_subset_ids_reaches is None: - extra_reaches_ids = [] - if not adapted_subset_ids_reaches: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] + if not adapted_subset_ids_reaches: + logger.debug( + "no adapted_subset_ids_reaches - so nothing to remove", + ) + else: + logger.debug( + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list( + set(subset_ids_reaches) - set(adapted_subset_ids_reaches) + ) + # 7. If extra_reaches then remove from adapted_subset_ids + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption logger.debug( - "no adapted_subset_ids_reaches - so nothing to remove", + "no extra reaches - so nothing to remove from adapted_subset_ids", ) else: logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", - ) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list( - set(subset_ids_reaches) - set(adapted_subset_ids_reaches) - ) - # 7. If extra_reaches then remove from adapted_subset_ids - if extra_reaches_ids is None: - if not extra_reaches_ids: - # list is empty - no need for adaption - logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", - ) - else: - logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", - ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection( - adapted_subset_ids, extra_reaches_ids - ) - # 8. get all id's of connected wastewater_structures - subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) - logger.info( - f"subset_wws_ids: {subset_wws_ids}", - ) - # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) - ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - - # 10. Show ws_off_sia405abwasser_list - logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - ) - # 11. take out ws_off_sia405abwasser_list from subset_wws_ids - subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) - logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", - ) - else: # flag_approach_urs = False - # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) - ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") - - # 3. Show ws_off_sia405abwasser_list - logger.debug( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - ) - - # 4. check if filtered - if filtered: - if ws_off_sia405abwasser_list: - # take out ws_off_sia405abwasser_list from selection - subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) - # else do nothing - else: - if ws_off_sia405abwasser_list: - # add all data of wastewater_structures to selection - subset_ids = add_to_selection( - subset_ids, get_ws_wn_ids("wastewater_structure") - ) - logger.debug( - f"subset_ids of all wws : {subset_ids}", + f"extra_reaches_ids: {extra_reaches_ids} found!", ) - # take out ws_off_sia405abwasser_list from selection - subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) - logger.debug( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids ) - # add reach_ids - # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) - # treat export as with a selection - filtered = True - - # else do nothing - - # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) - subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) - logger.debug( - f"subset_wws_ids: {subset_wws_ids}", - ) - subset_ids = add_to_selection(subset_ids, subset_wws_ids) - logger.debug( - f"subset_ids with wws : {subset_ids}", - ) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.info( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + + # 10. Show ws_off_sia405abwasser_list + logger.info( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.info( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) # also if not filtered we have to take out references to wwtp_structures else: From 2bf1406626486b50eaa58b5c38c42ce3dece9d86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:18:01 +0000 Subject: [PATCH 090/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index dfafa994..0680375b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -99,9 +99,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) - extra_reaches_ids = list( - set(subset_ids_reaches) - set(adapted_subset_ids_reaches) - ) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) # 7. If extra_reaches then remove from adapted_subset_ids if extra_reaches_ids is None: if not extra_reaches_ids: @@ -128,12 +126,12 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 10. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", # ) # also if not filtered we have to take out references to wwtp_structures @@ -144,18 +142,18 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 21. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", + # f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", # ) # 22. Get list of all wastewater_structures # subset_wws_ids = get_ws_ids("wastewater_structure") # logger.info( - # f"subset_wws_ids (non filtered) : {subset_wws_ids}", + # f"subset_wws_ids (non filtered) : {subset_wws_ids}", # ) # 23. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", + # f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", # ) logger.debug("Handling of wwtp_structures not needed with VSA-DSS") From af931c1fabe5b9c2d282ff68d7d0af4610172200 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 15:20:26 +0100 Subject: [PATCH 091/245] take out old approache kek export --- qgepqwat2ili/qgep/export.py | 174 +++++++++++++----------------------- 1 file changed, 63 insertions(+), 111 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 4642193a..09e0f1d0 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -58,124 +58,76 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e subset_wws_ids = [] if filtered: - flag_approach_urs = True - if flag_approach_urs: - # 2. Get all connected from wastewater_nodes of selected reaches - connected_from_wn_ids = get_connected_we_from_re(subset_ids) - # 3. Get all connected to wastewater_nodes of selected reaches - connected_to_wn_ids = get_connected_we_to_re(subset_ids) - # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to - connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) - # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids - adapted_subset_ids = [] - adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) - adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) - # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids - subset_ids_reaches = filter_reaches(subset_ids) - adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - if adapted_subset_ids_reaches is None: - extra_reaches_ids = [] - if not adapted_subset_ids_reaches: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = [] + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + if adapted_subset_ids_reaches is None: + extra_reaches_ids = [] + if not adapted_subset_ids_reaches: + logger.debug( + "no adapted_subset_ids_reaches - so nothing to remove", + ) + else: + logger.debug( + "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) + # https://www.geeksforgeeks.org/python-difference-two-lists/ + # First convert lists to sets + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + # replaced with code that first converts to sets + # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # Convert lists to sets and use the difference method + # c = list(set(a) - set(b)) + extra_reaches_ids = list( + set(subset_ids_reaches) - set(adapted_subset_ids_reaches) + ) + # 7. If extra_reaches then remove from adapted_subset_ids + if extra_reaches_ids is None: + if not extra_reaches_ids: + # list is empty - no need for adaption logger.debug( - "no adapted_subset_ids_reaches - so nothing to remove", + "no extra reaches - so nothing to remove from adapted_subset_ids", ) else: logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", - ) - # https://www.geeksforgeeks.org/python-difference-two-lists/ - # First convert lists to sets - # https://www.w3schools.com/python/ref_set_difference.asp - # x = {"apple", "banana", "cherry"} - # y = {"google", "microsoft", "apple"} - # z = x.difference(y) - # replaced with code that first converts to sets - # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) - # Convert lists to sets and use the difference method - # c = list(set(a) - set(b)) - extra_reaches_ids = list( - set(subset_ids_reaches) - set(adapted_subset_ids_reaches) - ) - # 7. If extra_reaches then remove from adapted_subset_ids - if extra_reaches_ids is None: - if not extra_reaches_ids: - # list is empty - no need for adaption - logger.debug( - "no extra reaches - so nothing to remove from adapted_subset_ids", - ) - else: - logger.debug( - f"extra_reaches_ids: {extra_reaches_ids} found!", - ) - # if len(extra_reaches_ids) > 0: - adapted_subset_ids = remove_from_selection( - adapted_subset_ids, extra_reaches_ids - ) - # 8. get all id's of connected wastewater_structures - subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) - logger.info( - f"subset_wws_ids: {subset_wws_ids}", - ) - # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) - ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - - # 10. Show ws_off_sia405abwasser_list - logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - ) - # 11. take out ws_off_sia405abwasser_list from subset_wws_ids - subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) - logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", - ) - else: # flag_approach_urs = False - # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) - ws_off_sia405abwasser_list = None - ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") - - # 3. Show ws_off_sia405abwasser_list - logger.debug( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", - ) - - # 4. check if filtered - if filtered: - if ws_off_sia405abwasser_list: - # take out ws_off_sia405abwasser_list from selection - subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) - # else do nothing - else: - if ws_off_sia405abwasser_list: - # add all data of wastewater_structures to selection - subset_ids = add_to_selection( - subset_ids, get_ws_wn_ids("wastewater_structure") - ) - logger.debug( - f"subset_ids of all wws : {subset_ids}", + f"extra_reaches_ids: {extra_reaches_ids} found!", ) - # take out ws_off_sia405abwasser_list from selection - subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) - logger.debug( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection( + adapted_subset_ids, extra_reaches_ids ) - # add reach_ids - # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) - # treat export as with a selection - filtered = True - - # else do nothing + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.info( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) - subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) - logger.debug( - f"subset_wws_ids: {subset_wws_ids}", - ) - subset_ids = add_to_selection(subset_ids, subset_wws_ids) - logger.debug( - f"subset_ids with wws : {subset_ids}", - ) + # 10. Show ws_off_sia405abwasser_list + logger.info( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.info( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) # also if not filtered we have to take out references to wwtp_structures else: From c53a8cb0837aa9dd7440922f519a9100d5bd4e78 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 15:23:01 +0100 Subject: [PATCH 092/245] readd get_ws_wn_ids, --- qgepqwat2ili/qgep/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 09e0f1d0..6a800308 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -16,6 +16,7 @@ get_connected_we_to_re, get_ws_ids, get_ws_selected_ww_networkelements, + get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger From c1698b29955d4129bde0914a49111b8b1b98f4bb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:23:17 +0000 Subject: [PATCH 093/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgep/export.py | 5 +---- qgepqwat2ili/qgepsia405/export.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 6a800308..f86c3c76 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -16,7 +16,6 @@ get_connected_we_to_re, get_ws_ids, get_ws_selected_ww_networkelements, - get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger @@ -93,9 +92,7 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) - extra_reaches_ids = list( - set(subset_ids_reaches) - set(adapted_subset_ids_reaches) - ) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) # 7. If extra_reaches then remove from adapted_subset_ids if extra_reaches_ids is None: if not extra_reaches_ids: diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 65722ad0..10ee6507 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -15,7 +15,6 @@ get_connected_we_to_re, get_ws_ids, get_ws_selected_ww_networkelements, - get_ws_wn_ids, remove_from_selection, ) from ..utils.various import logger @@ -92,9 +91,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) # Convert lists to sets and use the difference method # c = list(set(a) - set(b)) - extra_reaches_ids = list( - set(subset_ids_reaches) - set(adapted_subset_ids_reaches) - ) + extra_reaches_ids = list(set(subset_ids_reaches) - set(adapted_subset_ids_reaches)) # 7. If extra_reaches then remove from adapted_subset_ids if extra_reaches_ids is None: if not extra_reaches_ids: From 296176f09d3b73966874f9bf182cd232a9b795a1 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 16:13:03 +0100 Subject: [PATCH 094/245] add comment --- qgepqwat2ili/qgepdss/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 0680375b..46532513 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -868,6 +868,7 @@ def overflow_common(row): logger.info("Exporting QGEP.private -> ABWASSER.privat, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.private) + # always export all private for row in query: # AVAILABLE FIELDS IN QGEP.private From 7fc30649f1449eb2573ee1d53bffcfa4531b593d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 16:21:13 +0100 Subject: [PATCH 095/245] adapt filter throttle_shut_off_unit --- qgepqwat2ili/qgepdss/export.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 46532513..f8e03c9e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2763,10 +2763,7 @@ def overflow_common(row): # fk_control_center has also to be NOT considered if filtered: query = query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id - == qgep_model.throttle_shut_off_unit.fk_wastewater_node, + qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, ), ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From 77e0df1c2e5c26258a8d4189d3fab0a3328546ac Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 16:23:39 +0100 Subject: [PATCH 096/245] correct --- qgepqwat2ili/qgepdss/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f8e03c9e..91f6b96f 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2764,7 +2764,6 @@ def overflow_common(row): if filtered: query = query.join( qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, - ), ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From ed3d5cccd548b176f86583a927957964f2f68f14 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:23:57 +0000 Subject: [PATCH 097/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 91f6b96f..e2c874e7 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2763,7 +2763,9 @@ def overflow_common(row): # fk_control_center has also to be NOT considered if filtered: query = query.join( - qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, + qgep_model.wastewater_node, + qgep_model.wastewater_node.obj_id + == qgep_model.throttle_shut_off_unit.fk_wastewater_node, ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From d114496f4c2502dbb266160b8329fc16b6fee6d2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 16:37:03 +0100 Subject: [PATCH 098/245] logger.info(f"Selection query: {query.statement}") --- qgepqwat2ili/qgepdss/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 91f6b96f..a38a60eb 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2765,6 +2765,7 @@ def overflow_common(row): query = query.join( qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + logger.info(f"Selection query: {query.statement}") for row in query: # AVAILABLE FIELDS IN QGEP.throttle_shut_off_unit From 67bdfcfa98b0a3e9a490c89d6b024eeff8aba4c8 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 16:49:44 +0100 Subject: [PATCH 099/245] correct query control_center --- qgepqwat2ili/qgepdss/export.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 3e63c208..a912843e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1393,9 +1393,14 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.control_center) if filtered: - query = query.join(qgep_model.throttle_shut_off_unit, qgep_model.wastewater_node).filter( + query = query.join(qgep_model.throttle_shut_off_unit).join( + qgep_model.wastewater_node, + qgep_model.wastewater_node.obj_id + == qgep_model.throttle_shut_off_unit.fk_wastewater_node, + ).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + logger.info(f"Selection query: {query.statement}") for row in query: # AVAILABLE FIELDS IN QGEP.control_center From 7edddc151b2da378460a9a1236dfb911412865bc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:50:01 +0000 Subject: [PATCH 100/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index a912843e..f4589a4e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1393,12 +1393,14 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.control_center) if filtered: - query = query.join(qgep_model.throttle_shut_off_unit).join( - qgep_model.wastewater_node, - qgep_model.wastewater_node.obj_id - == qgep_model.throttle_shut_off_unit.fk_wastewater_node, - ).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) + query = ( + query.join(qgep_model.throttle_shut_off_unit) + .join( + qgep_model.wastewater_node, + qgep_model.wastewater_node.obj_id + == qgep_model.throttle_shut_off_unit.fk_wastewater_node, + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) logger.info(f"Selection query: {query.statement}") for row in query: From 46cc99837e57194703d53d470c3c0587e095f987 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 16:59:47 +0100 Subject: [PATCH 101/245] always export all profile_geometry --- qgepqwat2ili/qgepdss/export.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index a912843e..25ffede1 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1818,10 +1818,7 @@ def overflow_common(row): "Exporting QGEP.profile_geometry -> ABWASSER.rohrprofil_geometrie, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.profile_geometry) - if filtered: - query = query.join(qgep_model.pipe_profile, qgep_model.reach).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) - ) + # always export all profile_geometry for row in query: # AVAILABLE FIELDS IN QGEP.profile_geometry From a93919c077cb38022cdaf94afd54734f08cb4683 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:04:30 +0100 Subject: [PATCH 102/245] always export all hydr_geom_relation --- qgepqwat2ili/qgepdss/export.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 7b3a1e75..084fade4 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1854,10 +1854,7 @@ def overflow_common(row): "Exporting QGEP.hydr_geom_relation -> ABWASSER.hydr_geomrelation, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.hydr_geom_relation) - if filtered: - query = query.join(qgep_model.hydr_geometry, qgep_model.wastewater_node).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) - ) + # always export all hydr_geom_relation for row in query: # AVAILABLE FIELDS IN QGEP.hydr_geom_relation From be57c3b78b6b908c25498cf1d0beb1e32768d74f Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:12:11 +0100 Subject: [PATCH 103/245] correct filter mechanical_pretreatment --- qgepqwat2ili/qgepdss/export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 084fade4..8a194d49 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1888,9 +1888,11 @@ def overflow_common(row): "Exporting QGEP.mechanical_pretreatment -> ABWASSER.mechanischevorreinigung, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.mechanical_pretreatment) + # specify relation key - only directly to wastewater_structure if filtered: query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement + qgep_model.wastewater_structure, qgep_model.wastewater_node.obj_id + == qgep_model.mechanical_pretreatment.fk_wastewater_structure).join(qgep_model.wastewater_networkelement) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From 51ad7df1960bc6db67735b7fa10ac4fb7117c3e7 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:14:25 +0100 Subject: [PATCH 104/245] change relation --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 8a194d49..a710444f 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1891,8 +1891,8 @@ def overflow_common(row): # specify relation key - only directly to wastewater_structure if filtered: query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_node.obj_id - == qgep_model.mechanical_pretreatment.fk_wastewater_structure).join(qgep_model.wastewater_networkelement) + qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id,).join(qgep_model.wastewater_networkelement) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From 2a21bce8e5070ca18ac78797217fa54dc9b028a9 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:17:33 +0100 Subject: [PATCH 105/245] take out ) --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index a710444f..d07214a5 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1893,7 +1893,7 @@ def overflow_common(row): query = query.join( qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id,).join(qgep_model.wastewater_networkelement) - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: # AVAILABLE FIELDS IN QGEP.mechanical_pretreatment From 31e3413cad630cea17b5cf4094b4fbc311298744 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:19:57 +0100 Subject: [PATCH 106/245] correct --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index d07214a5..0d513ac4 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1891,8 +1891,8 @@ def overflow_common(row): # specify relation key - only directly to wastewater_structure if filtered: query = query.join( - qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id,).join(qgep_model.wastewater_networkelement) + qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id) + .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From 0d6bc676f67bf8cb1e96b7f774b9ba351da50169 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:23:12 +0100 Subject: [PATCH 107/245] formatting --- qgepqwat2ili/qgepdss/export.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 0d513ac4..c5306ee2 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1890,10 +1890,11 @@ def overflow_common(row): query = qgep_session.query(qgep_model.mechanical_pretreatment) # specify relation key - only directly to wastewater_structure if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id) + query = ( + query.join(qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id) .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.mechanical_pretreatment From 88fb0e01b23ab5889b6cabb3df9274ce5d40c997 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:23:30 +0000 Subject: [PATCH 108/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index c5306ee2..ce0d15e0 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1891,7 +1891,11 @@ def overflow_common(row): # specify relation key - only directly to wastewater_structure if filtered: query = ( - query.join(qgep_model.wastewater_structure, qgep_model.mechanical_pretreatment.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id) + query.join( + qgep_model.wastewater_structure, + qgep_model.mechanical_pretreatment.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) From 744e794e520c9c0fd5575434fba6f97705a1ad45 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:30:07 +0100 Subject: [PATCH 109/245] correct filter retention_body --- qgepqwat2ili/qgepdss/export.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index c5306ee2..2462839a 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1933,10 +1933,13 @@ def overflow_common(row): "Exporting QGEP.retention_body -> ABWASSER.retentionskoerper, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.retention_body) + # specify relation key - only directly to wastewater_structure if filtered: - query = query.join( - qgep_model.infiltration_installation, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join(qgep_model.infiltration_installation, qgep_model.retention_body.fk_wastewater_structure == qgep_model.infiltration_installation.obj_id) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.retention_body From f04da842d404e0f82c43012effcc89c01c5e6603 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:30:51 +0000 Subject: [PATCH 110/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 006fc0fd..c8520d5c 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1940,7 +1940,11 @@ def overflow_common(row): # specify relation key - only directly to wastewater_structure if filtered: query = ( - query.join(qgep_model.infiltration_installation, qgep_model.retention_body.fk_wastewater_structure == qgep_model.infiltration_installation.obj_id) + query.join( + qgep_model.infiltration_installation, + qgep_model.retention_body.fk_wastewater_structure + == qgep_model.infiltration_installation.obj_id, + ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) From cf3b23b8df8a8f4111537887fcd9e6c1ddb2136c Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:34:49 +0100 Subject: [PATCH 111/245] wrong fk --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 006fc0fd..70cc69ba 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1940,7 +1940,7 @@ def overflow_common(row): # specify relation key - only directly to wastewater_structure if filtered: query = ( - query.join(qgep_model.infiltration_installation, qgep_model.retention_body.fk_wastewater_structure == qgep_model.infiltration_installation.obj_id) + query.join(qgep_model.infiltration_installation, qgep_model.retention_body.fk_infiltration_installation == qgep_model.infiltration_installation.obj_id) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) From dd051bf5f2937af8381112f2db7a0cf0c1b512e1 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:48:35 +0100 Subject: [PATCH 112/245] correct filter electric_equipment --- qgepqwat2ili/qgepdss/export.py | 38 ++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 7f42602e..105dcb27 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1899,6 +1899,9 @@ def overflow_common(row): .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.mechanical_pretreatment @@ -1948,6 +1951,9 @@ def overflow_common(row): .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.retention_body @@ -2082,9 +2088,17 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.electric_equipment) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.electric_equipment @@ -2123,9 +2137,21 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.electromechanical_equipment) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.electromechanical_equipment From 313793ddf6a6e2506cbea8ac4686d94d17049270 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:49:02 +0000 Subject: [PATCH 113/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 105dcb27..e6f55205 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2091,7 +2091,8 @@ def overflow_common(row): query = ( query.join( qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) @@ -2144,7 +2145,8 @@ def overflow_common(row): query = ( query.join( qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From 777efa7bd1f5deb604e8b086b40fc8cdc3cffb1d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:52:04 +0100 Subject: [PATCH 114/245] correct filter hazard_source --- qgepqwat2ili/qgepdss/export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 105dcb27..c3c65cda 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2349,9 +2349,17 @@ def overflow_common(row): logger.info("Exporting QGEP.hazard_source -> ABWASSER.gefahrenquelle, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.hazard_source) if filtered: - query = query.join( - qgep_model.connection_object, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.connection_object, + qgep_model.hazard_source.fk_connection_object == qgep_model.connection_object.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.hazard_source From 9da3fe751ffe209814ef10da9229f2e025bcfe36 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:53:11 +0100 Subject: [PATCH 115/245] correct filter accident --- qgepqwat2ili/qgepdss/export.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 7f271f5e..d28f2ffc 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2397,11 +2397,17 @@ def overflow_common(row): logger.info("Exporting QGEP.accident -> ABWASSER.unfall, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.accident) if filtered: - query = query.join( - qgep_model.hazard_source, - qgep_model.connection_object, - qgep_model.wastewater_networkelement, + query = ( + query.join(qgep_model.hazard_source) + .join( + qgep_model.connection_object, + qgep_model.hazard_source.fk_connection_object == qgep_model.connection_object.obj_id, + ) + .join(qgep_model.wastewater_networkelement) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.accident From 4cbf74fc6539291d67a06b7cb00a3e4ccfc03c21 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:53:28 +0000 Subject: [PATCH 116/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index d28f2ffc..57891cae 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2354,7 +2354,8 @@ def overflow_common(row): query = ( query.join( qgep_model.connection_object, - qgep_model.hazard_source.fk_connection_object == qgep_model.connection_object.obj_id, + qgep_model.hazard_source.fk_connection_object + == qgep_model.connection_object.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) @@ -2401,7 +2402,8 @@ def overflow_common(row): query.join(qgep_model.hazard_source) .join( qgep_model.connection_object, - qgep_model.hazard_source.fk_connection_object == qgep_model.connection_object.obj_id, + qgep_model.hazard_source.fk_connection_object + == qgep_model.connection_object.obj_id, ) .join(qgep_model.wastewater_networkelement) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From 2ff3aee61114e46666d1e8a92fef7aba76fd6039 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:54:32 +0100 Subject: [PATCH 117/245] correct filter substance --- qgepqwat2ili/qgepdss/export.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index d28f2ffc..b29645fe 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2445,12 +2445,19 @@ def overflow_common(row): logger.info("Exporting QGEP.substance -> ABWASSER.stoff, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.substance) if filtered: - query = query.join( - qgep_model.hazard_source, - qgep_model.connection_object, - qgep_model.wastewater_networkelement, + + query = ( + query.join(qgep_model.hazard_source) + .join( + qgep_model.connection_object, + qgep_model.hazard_source.fk_connection_object == qgep_model.connection_object.obj_id, + ) + .join(qgep_model.wastewater_networkelement) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - for row in query: + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.substance From a1543e0b12f4fd94cb407c4f53b256a3d16e144d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 17:57:35 +0100 Subject: [PATCH 118/245] intend --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f8c577da..f2a76f8b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2459,7 +2459,7 @@ def overflow_common(row): # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") - for row in query: + for row in query: # AVAILABLE FIELDS IN QGEP.substance From bc7bdc9307f9e78f8e720c36ede4af9e772b13e5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:57:50 +0000 Subject: [PATCH 119/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f2a76f8b..2f8a4a6c 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2452,7 +2452,8 @@ def overflow_common(row): query.join(qgep_model.hazard_source) .join( qgep_model.connection_object, - qgep_model.hazard_source.fk_connection_object == qgep_model.connection_object.obj_id, + qgep_model.hazard_source.fk_connection_object + == qgep_model.connection_object.obj_id, ) .join(qgep_model.wastewater_networkelement) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From d52a7714072126b7cb3d17dcfe0a56759b5ecf0d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:37:36 +0100 Subject: [PATCH 120/245] correct filter measuring point --- qgepqwat2ili/qgepdss/export.py | 57 ++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f2a76f8b..65d23468 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2588,11 +2588,12 @@ def overflow_common(row): logger.info("Exporting QGEP.measuring_point -> ABWASSER.messstelle, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.measuring_point) - if filtered: +if filtered: query1 = query.join( - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ) + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ).join(QGEP.wastewater_networkelement) + # needs to add QGEP.wastewater_structure as waste_water_treatment_plant is a subclass of organisation that has a relation to wastewater_structure and then wastewater_networkelement # variant1 for query2 # query2=query.join( @@ -2610,23 +2611,47 @@ def overflow_common(row): # QGEP.wastewater_networkelement, # ) - # query2 via waste_water_treatment_plant TODO : Fix Mapping - query2 = query.join( - qgep_model.model_classes_tww_od.waste_water_treatment_plant, - qgep_model.model_classes_tww_od.wwtp_structure, - qgep_model.model_classes_tww_od.wastewater_networkelement, + + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) ) # only until VSA-DSS Release 2015 - query3 = query.join( - qgep_model.water_course_segment, - qgep_model.river, - qgep_model.sector_water_body, - qgep_model.discharge_point, - qgep_model.wastewater_networkelement, + query3 = ( + query.join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) ) query = query.union(query1, query2, query3) # query = query.union(query1, query3) - query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.measuring_point From a332fa9dee89260332ec0666f2a6f78da16a5564 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:41:09 +0100 Subject: [PATCH 121/245] correct filter measuring_device --- qgepqwat2ili/qgepdss/export.py | 77 ++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 952089d6..6b0c9dd2 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2589,7 +2589,7 @@ def overflow_common(row): logger.info("Exporting QGEP.measuring_point -> ABWASSER.messstelle, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.measuring_point) -if filtered: + if filtered: query1 = query.join( QGEP.wastewater_structure, QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, @@ -2697,18 +2697,69 @@ def overflow_common(row): logger.info("Exporting QGEP.measuring_device -> ABWASSER.messgeraet, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.measuring_device) if filtered: - query = query.join( - qgep_model.measuring_point, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # or does not work with this - currently do not support - # QGEP.wastewater_networkelement, - # or_( - # (QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # ) + # query = query.join( + # QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement + # ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + + query1 = ( + query.join( + QGEP.measuring_point, + QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.measuring_point, + QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # only until VSA-DSS Release 2015 + query3 = ( + query.join( + QGEP.measuring_point, + QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + query = query.union(query1, query2, query3) + # query = query.union(query1, query3) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: From 392936fd9d7bea9e0ce93fae38949cf95a9a8153 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:46:10 +0100 Subject: [PATCH 122/245] correct measurment_series --- qgepqwat2ili/qgepdss/export.py | 164 ++++++++++++++++++++++----------- 1 file changed, 108 insertions(+), 56 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 6b0c9dd2..051d6c1b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2591,9 +2591,9 @@ def overflow_common(row): query = qgep_session.query(qgep_model.measuring_point) if filtered: query1 = query.join( - QGEP.wastewater_structure, - QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, - ).join(QGEP.wastewater_networkelement) + qgep_model.wastewater_structure, + qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ).join(qgep_model.wastewater_networkelement) # needs to add QGEP.wastewater_structure as waste_water_treatment_plant is a subclass of organisation that has a relation to wastewater_structure and then wastewater_networkelement # variant1 for query2 @@ -2616,40 +2616,40 @@ def overflow_common(row): # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation query2 = ( query.join( - QGEP.waste_water_treatment_plant, - QGEP.measuring_point.fk_waste_water_treatment_plant - == QGEP.waste_water_treatment_plant.obj_id, + qgep_model.waste_water_treatment_plant, + qgep_model.measuring_point.fk_waste_water_treatment_plant + == qgep_model.waste_water_treatment_plant.obj_id, ) .join( - QGEP.wastewater_structure, - QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure, + qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) # only until VSA-DSS Release 2015 query3 = ( query.join( - QGEP.water_course_segment, - QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + qgep_model.water_course_segment, + qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, ) .join( - QGEP.river, + qgep_model.river, # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) - QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + qgep_model.water_course_segment.fk_watercourse == qgep_model.river.obj_id, ) .join( - QGEP.sector_water_body, - QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + qgep_model.sector_water_body, + qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, ) .join( - QGEP.discharge_point, - QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + qgep_model.discharge_point, + qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) query = query.union(query1, query2, query3) # query = query.union(query1, query3) - query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -2703,67 +2703,67 @@ def overflow_common(row): query1 = ( query.join( - QGEP.measuring_point, - QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + qgep_model.measuring_point, + qgep_model.measuring_device.fk_measuring_point == qgep_model.measuring_point.obj_id, ) .join( - QGEP.wastewater_structure, - QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + qgep_model.wastewater_structure, + qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation query2 = ( query.join( - QGEP.measuring_point, - QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + qgep_model.measuring_point, + qgep_model.measuring_device.fk_measuring_point == qgep_model.measuring_point.obj_id, ) .join( - QGEP.waste_water_treatment_plant, - QGEP.measuring_point.fk_waste_water_treatment_plant - == QGEP.waste_water_treatment_plant.obj_id, + qgep_model.waste_water_treatment_plant, + qgep_model.measuring_point.fk_waste_water_treatment_plant + == qgep_model.waste_water_treatment_plant.obj_id, ) .join( - QGEP.wastewater_structure, - QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure, + qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) # only until VSA-DSS Release 2015 query3 = ( query.join( - QGEP.measuring_point, - QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + qgep_model.measuring_point, + qgep_model.measuring_device.fk_measuring_point == qgep_model.measuring_point.obj_id, ) .join( - QGEP.water_course_segment, - QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + qgep_model.water_course_segment, + qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, ) .join( - QGEP.river, + qgep_model.river, # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) - QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + qgep_model.water_course_segment.fk_watercourse == QGEP.river.obj_id, ) .join( - QGEP.sector_water_body, - QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + qgep_model.sector_water_body, + qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, ) .join( - QGEP.discharge_point, - QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + qgep_model.discharge_point, + qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) query = query.union(query1, query2, query3) # query = query.union(query1, query3) - query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") for row in query: - # AVAILABLE FIELDS IN QGEP.measuring_device + # AVAILABLE FIELDS IN qgep_model.measuring_device # --- measuring_device --- # to do e.g. fk_dataowner, fk_provider, height_width_ratio, identifier, last_modification, obj_id, profile_type, remark @@ -2797,17 +2797,69 @@ def overflow_common(row): logger.info("Exporting QGEP.measurement_series -> ABWASSER.messreihe, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.measurement_series) if filtered: - query = query.join( - qgep_model.measuring_point, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # QGEP.wastewater_networkelement, - # or_( - # (QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # ) + + # query = query.join( + # qgep_model.measuring_point, qgep_model.wastewater_structure, qgep_model.wastewater_networkelement + # ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query1 = ( + query.join( + qgep_model.measuring_point, + qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + ) + .join( + qgep_model.wastewater_structure, + qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + ) + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + qgep_model.measuring_point, + qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + ) + .join( + qgep_model.waste_water_treatment_plant, + qgep_model.measuring_point.fk_waste_water_treatment_plant + == qgep_model.waste_water_treatment_plant.obj_id, + ) + .join( + qgep_model.wastewater_structure, + qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + ) + # only until VSA-DSS Release 2015 + query3 = ( + query.join( + qgep_model.measuring_point, + qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + ) + .join( + qgep_model.water_course_segment, + qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, + ) + .join( + qgep_model.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + qgep_model.water_course_segment.fk_watercourse == qgep_model.river.obj_id, + ) + .join( + qgep_model.sector_water_body, + qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, + ) + .join( + qgep_model.discharge_point, + qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + ) + query = query.union(query1, query2, query3) + # query = query.union(query1, query3) + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.measurement_series From b76f611552e896c9b2b24235b3880c5715a94a95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:47:08 +0000 Subject: [PATCH 123/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 63 ++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 051d6c1b..8ad9494d 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2592,7 +2592,8 @@ def overflow_common(row): if filtered: query1 = query.join( qgep_model.wastewater_structure, - qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.measuring_point.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ).join(qgep_model.wastewater_networkelement) # needs to add QGEP.wastewater_structure as waste_water_treatment_plant is a subclass of organisation that has a relation to wastewater_structure and then wastewater_networkelement @@ -2622,7 +2623,8 @@ def overflow_common(row): ) .join( qgep_model.wastewater_structure, - qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure.fk_owner + == qgep_model.waste_water_treatment_plant.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2630,7 +2632,8 @@ def overflow_common(row): query3 = ( query.join( qgep_model.water_course_segment, - qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, + qgep_model.measuring_point.fk_water_course_segment + == qgep_model.water_course_segment.obj_id, ) .join( qgep_model.river, @@ -2639,11 +2642,13 @@ def overflow_common(row): ) .join( qgep_model.sector_water_body, - qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, + qgep_model.sector_water_body.fk_surface_water_bodies + == qgep_model.sector_water_body.obj_id, ) .join( qgep_model.discharge_point, - qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, + qgep_model.discharge_point.fk_sector_water_body + == qgep_model.sector_water_body.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2704,11 +2709,13 @@ def overflow_common(row): query1 = ( query.join( qgep_model.measuring_point, - qgep_model.measuring_device.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measuring_device.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.wastewater_structure, - qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.measuring_point.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2716,7 +2723,8 @@ def overflow_common(row): query2 = ( query.join( qgep_model.measuring_point, - qgep_model.measuring_device.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measuring_device.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.waste_water_treatment_plant, @@ -2725,7 +2733,8 @@ def overflow_common(row): ) .join( qgep_model.wastewater_structure, - qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure.fk_owner + == qgep_model.waste_water_treatment_plant.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2733,11 +2742,13 @@ def overflow_common(row): query3 = ( query.join( qgep_model.measuring_point, - qgep_model.measuring_device.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measuring_device.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.water_course_segment, - qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, + qgep_model.measuring_point.fk_water_course_segment + == qgep_model.water_course_segment.obj_id, ) .join( qgep_model.river, @@ -2746,11 +2757,13 @@ def overflow_common(row): ) .join( qgep_model.sector_water_body, - qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, + qgep_model.sector_water_body.fk_surface_water_bodies + == qgep_model.sector_water_body.obj_id, ) .join( qgep_model.discharge_point, - qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, + qgep_model.discharge_point.fk_sector_water_body + == qgep_model.sector_water_body.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2804,11 +2817,13 @@ def overflow_common(row): query1 = ( query.join( qgep_model.measuring_point, - qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measurement_series.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.wastewater_structure, - qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.measuring_point.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2816,7 +2831,8 @@ def overflow_common(row): query2 = ( query.join( qgep_model.measuring_point, - qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measurement_series.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.waste_water_treatment_plant, @@ -2825,7 +2841,8 @@ def overflow_common(row): ) .join( qgep_model.wastewater_structure, - qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure.fk_owner + == qgep_model.waste_water_treatment_plant.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2833,11 +2850,13 @@ def overflow_common(row): query3 = ( query.join( qgep_model.measuring_point, - qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measurement_series.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.water_course_segment, - qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, + qgep_model.measuring_point.fk_water_course_segment + == qgep_model.water_course_segment.obj_id, ) .join( qgep_model.river, @@ -2846,11 +2865,13 @@ def overflow_common(row): ) .join( qgep_model.sector_water_body, - qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, + qgep_model.sector_water_body.fk_surface_water_bodies + == qgep_model.sector_water_body.obj_id, ) .join( qgep_model.discharge_point, - qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, + qgep_model.discharge_point.fk_sector_water_body + == qgep_model.sector_water_body.obj_id, ) .join(qgep_model.wastewater_networkelement) ) From 2a8254b58daaa9eb140acb59e627a7ee799239f0 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:48:24 +0100 Subject: [PATCH 124/245] correct filter measurment_result --- qgepqwat2ili/qgepdss/export.py | 92 ++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 051d6c1b..4ded7dff 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2899,21 +2899,85 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.measurement_result) if filtered: - query = query.join( - qgep_model.measurement_series, - qgep_model.measuring_point, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # or_( - # (QGEP.measurement_series, QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measurement_series, QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measurement_series, QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # (QGEP.measuring_device, QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measuring_device, QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measuring_device, QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # ) + # query = query.join( + # QGEP.measurement_series, + # QGEP.measuring_point, + # QGEP.wastewater_structure, + # QGEP.wastewater_networkelement, + # ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + query1 = ( + query.join( + QGEP.measurement_series, + QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + ) + .join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.measurement_series, + QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + ) + .join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # only until VSA-DSS Release 2015 + query3 = ( + query.join( + QGEP.measurement_series, + QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + ) + .join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query4 not implemented via measuring_device + query = query.union(query1, query2, query3) + # query = query.union(query1, query3) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: From 735dca9f4fa12d03a4d2e02f208faad81d58cb41 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:50:48 +0100 Subject: [PATCH 125/245] correction --- qgepqwat2ili/qgepdss/export.py | 62 +++++++++++++++++----------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index e5c69cb0..83e8f684 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2929,73 +2929,73 @@ def overflow_common(row): # ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) query1 = ( query.join( - QGEP.measurement_series, - QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + qgep_model.measurement_series, + qgep_model.measurement_result.fk_measurement_series == qgep_model.measurement_series.obj_id, ) .join( - QGEP.measuring_point, - QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + qgep_model.measuring_point, + qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, ) .join( - QGEP.wastewater_structure, - QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + qgep_model.wastewater_structure, + qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation query2 = ( query.join( - QGEP.measurement_series, - QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + qgep_model.measurement_series, + qgep_model.measurement_result.fk_measurement_series == qgep_model.measurement_series.obj_id, ) .join( - QGEP.measuring_point, - QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + qgep_model.measuring_point, + qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, ) .join( - QGEP.waste_water_treatment_plant, - QGEP.measuring_point.fk_waste_water_treatment_plant - == QGEP.waste_water_treatment_plant.obj_id, + qgep_model.waste_water_treatment_plant, + qgep_model.measuring_point.fk_waste_water_treatment_plant + == qgep_model.waste_water_treatment_plant.obj_id, ) .join( - QGEP.wastewater_structure, - QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure, + qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) # only until VSA-DSS Release 2015 query3 = ( query.join( - QGEP.measurement_series, - QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + qgep_model.measurement_series, + qgep_model.measurement_result.fk_measurement_series == qgep_model.measurement_series.obj_id, ) .join( - QGEP.measuring_point, - QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + qgep_model.measuring_point, + qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, ) .join( - QGEP.water_course_segment, - QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + qgep_model.water_course_segment, + qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, ) .join( - QGEP.river, + qgep_model.river, # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) - QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + qgep_model.water_course_segment.fk_watercourse == qgep_model.river.obj_id, ) .join( - QGEP.sector_water_body, - QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + qgep_model.sector_water_body, + qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, ) .join( - QGEP.discharge_point, - QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + qgep_model.discharge_point, + qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, ) - .join(QGEP.wastewater_networkelement) + .join(qgep_model.wastewater_networkelement) ) # query4 not implemented via measuring_device query = query.union(query1, query2, query3) # query = query.union(query1, query3) - query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From 52107069f86282214c3ea932b12d6518e2aa4c35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:51:03 +0000 Subject: [PATCH 126/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 83e8f684..37f4236d 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2930,15 +2930,18 @@ def overflow_common(row): query1 = ( query.join( qgep_model.measurement_series, - qgep_model.measurement_result.fk_measurement_series == qgep_model.measurement_series.obj_id, + qgep_model.measurement_result.fk_measurement_series + == qgep_model.measurement_series.obj_id, ) .join( qgep_model.measuring_point, - qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measurement_series.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.wastewater_structure, - qgep_model.measuring_point.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.measuring_point.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2946,11 +2949,13 @@ def overflow_common(row): query2 = ( query.join( qgep_model.measurement_series, - qgep_model.measurement_result.fk_measurement_series == qgep_model.measurement_series.obj_id, + qgep_model.measurement_result.fk_measurement_series + == qgep_model.measurement_series.obj_id, ) .join( qgep_model.measuring_point, - qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measurement_series.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.waste_water_treatment_plant, @@ -2959,7 +2964,8 @@ def overflow_common(row): ) .join( qgep_model.wastewater_structure, - qgep_model.wastewater_structure.fk_owner == qgep_model.waste_water_treatment_plant.obj_id, + qgep_model.wastewater_structure.fk_owner + == qgep_model.waste_water_treatment_plant.obj_id, ) .join(qgep_model.wastewater_networkelement) ) @@ -2967,15 +2973,18 @@ def overflow_common(row): query3 = ( query.join( qgep_model.measurement_series, - qgep_model.measurement_result.fk_measurement_series == qgep_model.measurement_series.obj_id, + qgep_model.measurement_result.fk_measurement_series + == qgep_model.measurement_series.obj_id, ) .join( qgep_model.measuring_point, - qgep_model.measurement_series.fk_measuring_point == qgep_model.measuring_point.obj_id, + qgep_model.measurement_series.fk_measuring_point + == qgep_model.measuring_point.obj_id, ) .join( qgep_model.water_course_segment, - qgep_model.measuring_point.fk_water_course_segment == qgep_model.water_course_segment.obj_id, + qgep_model.measuring_point.fk_water_course_segment + == qgep_model.water_course_segment.obj_id, ) .join( qgep_model.river, @@ -2984,11 +2993,13 @@ def overflow_common(row): ) .join( qgep_model.sector_water_body, - qgep_model.sector_water_body.fk_surface_water_bodies == qgep_model.sector_water_body.obj_id, + qgep_model.sector_water_body.fk_surface_water_bodies + == qgep_model.sector_water_body.obj_id, ) .join( qgep_model.discharge_point, - qgep_model.discharge_point.fk_sector_water_body == qgep_model.sector_water_body.obj_id, + qgep_model.discharge_point.fk_sector_water_body + == qgep_model.sector_water_body.obj_id, ) .join(qgep_model.wastewater_networkelement) ) From 9efce0720a00c0c661593d3b0e4131a454b2bf66 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:51:38 +0100 Subject: [PATCH 127/245] correction --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 83e8f684..797f7d16 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2753,7 +2753,7 @@ def overflow_common(row): .join( qgep_model.river, # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) - qgep_model.water_course_segment.fk_watercourse == QGEP.river.obj_id, + qgep_model.water_course_segment.fk_watercourse == qgep_model.river.obj_id, ) .join( qgep_model.sector_water_body, From 80acbbfd9d76cfb7bc9cf8b6a653093bf7a345f4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:56:24 +0100 Subject: [PATCH 128/245] correct filter backflow_prevention --- qgepqwat2ili/qgepdss/export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index a535ef70..5151a694 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3311,9 +3311,17 @@ def overflow_common(row): query = qgep_session.query(qgep_model.backflow_prevention) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.backflow_prevention From 346d53a5c817f20ddbd42460c3448a608488fef4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 18:57:05 +0100 Subject: [PATCH 129/245] correct --- qgepqwat2ili/qgepdss/export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 5151a694..b9259849 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3313,11 +3313,11 @@ def overflow_common(row): if filtered: query = ( query.join( - QGEP.wastewater_structure, - QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(QGEP.wastewater_networkelement) - .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) # add sql statement to logger statement = query.statement From 468260d685015e4e1e7534b9c07b28a775dc1aec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:57:20 +0000 Subject: [PATCH 130/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b9259849..7f794068 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3314,7 +3314,8 @@ def overflow_common(row): query = ( query.join( qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From 4c50d61991564cf9d038caf636c3cd5ede12aa31 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:00:08 +0100 Subject: [PATCH 131/245] correct filter solids_retention --- qgepqwat2ili/qgepdss/export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b9259849..5e0f52fa 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3362,9 +3362,17 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.solids_retention) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.solids_retention From aeb03415b7a4af5354ff9b9f157ae02795d071c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:01:06 +0000 Subject: [PATCH 132/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b17e4387..8db35a63 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3366,7 +3366,8 @@ def overflow_common(row): query = ( query.join( qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From f0af16f09058991490b004c0f9885e0a272481d5 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:02:07 +0100 Subject: [PATCH 133/245] correct filter tank_cleaning --- qgepqwat2ili/qgepdss/export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b17e4387..2b379e8b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3412,9 +3412,17 @@ def overflow_common(row): logger.info("Exporting QGEP.tank_cleaning -> ABWASSER.beckenreinigung, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.tank_cleaning) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.tank_cleaning From 7e62c6e2564fe8881e9334ee6d3536b1787312e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:02:32 +0000 Subject: [PATCH 134/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f054447a..899fb0a0 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3416,7 +3416,8 @@ def overflow_common(row): query = ( query.join( qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From 08aa9bf80bfeec6c7163915cda48fcdc3151ad58 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:03:38 +0100 Subject: [PATCH 135/245] correct filter tank_emptying --- qgepqwat2ili/qgepdss/export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f054447a..44765f96 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3463,9 +3463,17 @@ def overflow_common(row): query = qgep_session.query(qgep_model.tank_emptying) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.tank_emptying From fd9690591125094ab913aaf8dccb91f77c22370e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:04:03 +0000 Subject: [PATCH 136/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 132ea2a5..acb8e203 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3467,7 +3467,8 @@ def overflow_common(row): query = ( query.join( qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join(qgep_model.wastewater_networkelement) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) From d01a78f14a933da19ebb004265f953f26833699a Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:09:34 +0100 Subject: [PATCH 137/245] correct filter maintenance_event --- qgepqwat2ili/qgepdss/export.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index acb8e203..7ea1bd9e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3604,13 +3604,25 @@ def overflow_common(row): "Exporting QGEP.maintenance_event -> ABWASSER.maintenance_event, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.maintenance_event) - # to check if join is correct like this n:m re_maintenance_event_wastewater_structure + # explicit join for n:m re_maintenance_event_wastewater_structure if filtered: - query = query.join( - qgep_model.re_maintenance_event_wastewater_structure, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.re_maintenance_event_wastewater_structure, + qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event + == qgep_model.maintenance_event.obj_id, + ) + .join( + qgep_model.wastewater_structure, + qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ) + .join(qgep_model.wastewater_networkelement) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.maintenance_event From 4f4d7d2d2a41d071b1fdd11437b2dfe15a3f35a5 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:18:10 +0100 Subject: [PATCH 138/245] correct filter hydr_geom_relation --- qgepqwat2ili/qgepdss/export.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 7ea1bd9e..b757f097 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1854,7 +1854,11 @@ def overflow_common(row): "Exporting QGEP.hydr_geom_relation -> ABWASSER.hydr_geomrelation, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.hydr_geom_relation) - # always export all hydr_geom_relation + # always export all hydr_geom_relation - does not work -> fk_errors + if filtered: + query = query.join(qgep_model.fk_hydr_geometry).join(qgep_model.wastewater_node).filter( + qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) + ) for row in query: # AVAILABLE FIELDS IN QGEP.hydr_geom_relation From 4711d96000a5c521e79eb9664f54af79120bcc83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:18:26 +0000 Subject: [PATCH 139/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b757f097..48a7a2aa 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1856,8 +1856,10 @@ def overflow_common(row): query = qgep_session.query(qgep_model.hydr_geom_relation) # always export all hydr_geom_relation - does not work -> fk_errors if filtered: - query = query.join(qgep_model.fk_hydr_geometry).join(qgep_model.wastewater_node).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) + query = ( + query.join(qgep_model.fk_hydr_geometry) + .join(qgep_model.wastewater_node) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) for row in query: From 4a09df79993b1c453ea0b1f8a52ac5a0b1e215e4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:21:23 +0100 Subject: [PATCH 140/245] correction --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b757f097..31954f0d 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1856,7 +1856,7 @@ def overflow_common(row): query = qgep_session.query(qgep_model.hydr_geom_relation) # always export all hydr_geom_relation - does not work -> fk_errors if filtered: - query = query.join(qgep_model.fk_hydr_geometry).join(qgep_model.wastewater_node).filter( + query = query.join(qgep_model.hydr_geometry).join(qgep_model.wastewater_node).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) for row in query: From 6bc2748e97815f3e9b59be03727026bde3be3663 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:23:07 +0000 Subject: [PATCH 141/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 31954f0d..b92d018b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1856,8 +1856,10 @@ def overflow_common(row): query = qgep_session.query(qgep_model.hydr_geom_relation) # always export all hydr_geom_relation - does not work -> fk_errors if filtered: - query = query.join(qgep_model.hydr_geometry).join(qgep_model.wastewater_node).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) + query = ( + query.join(qgep_model.hydr_geometry) + .join(qgep_model.wastewater_node) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) for row in query: From 946d2d909ec16250e878f93fd6bc6c0569130694 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:30:20 +0100 Subject: [PATCH 142/245] correct filter re_maintenance_event_wastewater_structure --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 31954f0d..5b19f1a8 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3782,7 +3782,7 @@ def overflow_common(row): query = qgep_session.query(qgep_model.re_maintenance_event_wastewater_structure) if filtered: query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement + qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure = qgep_model.wastewater_structure.obj_id).join(qgep_model.wastewater_networkelement,qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event = qgep_model.maintenance_event.obj_id) ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) for row in query: From ea28689b384c1195c3decd5597a75c8f9d8be24f Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:32:42 +0100 Subject: [PATCH 143/245] correct ) --- qgepqwat2ili/qgepdss/export.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 500468ad..d824b9e1 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3783,9 +3783,15 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.re_maintenance_event_wastewater_structure) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure = qgep_model.wastewater_structure.obj_id).join(qgep_model.wastewater_networkelement,qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event = qgep_model.maintenance_event.obj_id) - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure = qgep_model.wastewater_structure.obj_id + ) + .join( + qgep_model.wastewater_networkelement,qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event = qgep_model.maintenance_event.obj_id + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) for row in query: # AVAILABLE FIELDS IN QGEP.maintenance_event_wastewater_structure From 741c93af64aa0292f01ed95726e8f99bc4d93be0 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:34:27 +0100 Subject: [PATCH 144/245] correct == --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index d824b9e1..d4534ea3 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3785,10 +3785,10 @@ def overflow_common(row): if filtered: query = ( query.join( - qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure = qgep_model.wastewater_structure.obj_id + qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id ) .join( - qgep_model.wastewater_networkelement,qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event = qgep_model.maintenance_event.obj_id + qgep_model.wastewater_networkelement,qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event == qgep_model.maintenance_event.obj_id ) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) From e38153af6b17ea256e5e7f3e376dd86090782ed1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:35:36 +0000 Subject: [PATCH 145/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index d4534ea3..bd381c3c 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3785,12 +3785,16 @@ def overflow_common(row): if filtered: query = ( query.join( - qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id + qgep_model.wastewater_structure, + qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, ) .join( - qgep_model.wastewater_networkelement,qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event == qgep_model.maintenance_event.obj_id + qgep_model.wastewater_networkelement, + qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event + == qgep_model.maintenance_event.obj_id, ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) for row in query: From 230e130d3aec0b275a9b39a0dcaf1c694e1ce83e Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:40:58 +0100 Subject: [PATCH 146/245] correct query --- qgepqwat2ili/qgepdss/export.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index bd381c3c..ba95346d 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3784,6 +3784,11 @@ def overflow_common(row): query = qgep_session.query(qgep_model.re_maintenance_event_wastewater_structure) if filtered: query = ( + .join( + qgep_model.maintenance_event, + qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event + == qgep_model.maintenance_event.obj_id, + ) query.join( qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure @@ -3791,8 +3796,6 @@ def overflow_common(row): ) .join( qgep_model.wastewater_networkelement, - qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event - == qgep_model.maintenance_event.obj_id, ) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) From f21c26e589f461e3ef923383dcc2a39f1f2d0916 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:41:55 +0100 Subject: [PATCH 147/245] correct query.join --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index ba95346d..46a6f00a 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3784,12 +3784,12 @@ def overflow_common(row): query = qgep_session.query(qgep_model.re_maintenance_event_wastewater_structure) if filtered: query = ( - .join( + query.join( qgep_model.maintenance_event, qgep_model.re_maintenance_event_wastewater_structure.fk_maintenance_event == qgep_model.maintenance_event.obj_id, ) - query.join( + .join( qgep_model.wastewater_structure, qgep_model.re_maintenance_event_wastewater_structure.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, From 12fe804a18f68b929d8d0abd2051421668421060 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:56:43 +0100 Subject: [PATCH 148/245] logger.debug added with query statement --- qgepqwat2ili/qgepdss/export.py | 156 ++++++++++++++++++++++++--------- 1 file changed, 117 insertions(+), 39 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 46a6f00a..f577e4bc 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -293,6 +293,10 @@ def overflow_common(row): # only export explicitly specified mutation objects if filtered if filtered: query = query.filter(qgep_model.mutation.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.mutation @@ -912,7 +916,10 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) - for row in query: + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.discharge_point # --- wastewater_structure --- @@ -964,6 +971,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.infiltration_installation @@ -1017,6 +1027,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.wwtp_structure @@ -1402,7 +1415,9 @@ def overflow_common(row): ) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) - logger.info(f"Selection query: {query.statement}") + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.control_center @@ -1737,6 +1752,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_node).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.hydr_geometry @@ -1780,6 +1798,9 @@ def overflow_common(row): query = qgep_session.query(qgep_model.wastewater_node) if filtered: query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.wastewater_node @@ -1861,6 +1882,9 @@ def overflow_common(row): .join(qgep_model.wastewater_node) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.hydr_geom_relation @@ -2202,6 +2226,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.building @@ -2242,6 +2269,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.reservoir @@ -2282,6 +2312,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.individual_surface @@ -2322,6 +2355,9 @@ def overflow_common(row): query = query.join(qgep_model.wastewater_networkelement).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.fountain @@ -2501,7 +2537,8 @@ def overflow_common(row): logger.info("Exporting QGEP.catchment_area -> ABWASSER.einzugsgebiet, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.catchment_area) if filtered: - query = query.join( + query = ( + query.join( qgep_model.wastewater_networkelement, or_( qgep_model.wastewater_networkelement.obj_id @@ -2513,7 +2550,11 @@ def overflow_common(row): qgep_model.wastewater_networkelement.obj_id == qgep_model.catchment_area.fk_wastewater_networkelement_ww_current, ), - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.catchment_area @@ -3059,12 +3100,15 @@ def overflow_common(row): # sqlalchemy.exc.InvalidRequestError: Don't know how to join to . Please use the .select_from() method to establish an explicit left side, as well as providing an explcit ON clause if not present already to help resolve the ambiguity. # fk_control_center has also to be NOT considered if filtered: - query = query.join( - qgep_model.wastewater_node, - qgep_model.wastewater_node.obj_id - == qgep_model.throttle_shut_off_unit.fk_wastewater_node, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - logger.info(f"Selection query: {query.statement}") + query = ( + query.join( + qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.throttle_shut_off_unit @@ -3113,13 +3157,20 @@ def overflow_common(row): query = qgep_session.query(qgep_model.prank_weir) # to check if fk_overflow_char also has to be considered if filtered: - query = query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, - ), - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, + ), + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.prank_weir @@ -3159,13 +3210,19 @@ def overflow_common(row): query = qgep_session.query(qgep_model.pump) # to check if fk_overflow_char also has to be considered if filtered: - query = query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, - ), - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, + ), + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.pump @@ -3209,13 +3266,19 @@ def overflow_common(row): query = qgep_session.query(qgep_model.leapingweir) # to check if fk_overflow_char also has to be considered if filtered: - query = query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, - ), - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, + ), + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.leapingweir @@ -3255,14 +3318,20 @@ def overflow_common(row): query = qgep_session.query(qgep_model.hydraulic_char_data) # side fk_overflow_char not considered in filter query if filtered: - query = query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id - == qgep_model.hydraulic_char_data.fk_wastewater_node, - # fk_primary_direction only added with VSA-DSS 2020 - ), - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id + == qgep_model.hydraulic_char_data.fk_wastewater_node, + # fk_primary_direction only added with VSA-DSS 2020 + ), + ) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.hydraulic_char_data @@ -3526,6 +3595,9 @@ def overflow_common(row): query = query.join(qgep_model.catchment_area).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.param_ca_general @@ -3569,6 +3641,9 @@ def overflow_common(row): query = query.join(qgep_model.catchment_area).filter( qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.param_ca_mouse1 @@ -3799,6 +3874,9 @@ def overflow_common(row): ) .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.maintenance_event_wastewater_structure From f488955f25c9d273d002c1ef740d7df488d84d47 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 19:58:19 +0100 Subject: [PATCH 149/245] correct ) --- qgepqwat2ili/qgepdss/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f577e4bc..f44bc2b2 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -293,7 +293,6 @@ def overflow_common(row): # only export explicitly specified mutation objects if filtered if filtered: query = query.filter(qgep_model.mutation.obj_id.in_(subset_ids)) - ) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From f9fa0e33487c1a6d28edf5216a7af0cb9bf17734 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 13 Dec 2024 20:00:01 +0100 Subject: [PATCH 150/245] more ) --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f44bc2b2..f4e9655e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -918,7 +918,7 @@ def overflow_common(row): # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") - for row in query: + for row in query: # AVAILABLE FIELDS IN QGEP.discharge_point # --- wastewater_structure --- @@ -3103,7 +3103,7 @@ def overflow_common(row): query.join( qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) + .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) ) # add sql statement to logger statement = query.statement From e7c44c1ade6c71597062fc147499aab60f7965b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 19:02:08 +0000 Subject: [PATCH 151/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 87 ++++++++++++++-------------------- 1 file changed, 36 insertions(+), 51 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index f4e9655e..4127a08e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2536,8 +2536,7 @@ def overflow_common(row): logger.info("Exporting QGEP.catchment_area -> ABWASSER.einzugsgebiet, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.catchment_area) if filtered: - query = ( - query.join( + query = query.join( qgep_model.wastewater_networkelement, or_( qgep_model.wastewater_networkelement.obj_id @@ -2549,8 +2548,7 @@ def overflow_common(row): qgep_model.wastewater_networkelement.obj_id == qgep_model.catchment_area.fk_wastewater_networkelement_ww_current, ), - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - ) + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3099,12 +3097,11 @@ def overflow_common(row): # sqlalchemy.exc.InvalidRequestError: Don't know how to join to . Please use the .select_from() method to establish an explicit left side, as well as providing an explcit ON clause if not present already to help resolve the ambiguity. # fk_control_center has also to be NOT considered if filtered: - query = ( - query.join( - qgep_model.wastewater_node, qgep_model.wastewater_node.obj_id == qgep_model.throttle_shut_off_unit.fk_wastewater_node, - ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - ) + query = query.join( + qgep_model.wastewater_node, + qgep_model.wastewater_node.obj_id + == qgep_model.throttle_shut_off_unit.fk_wastewater_node, + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3156,16 +3153,13 @@ def overflow_common(row): query = qgep_session.query(qgep_model.prank_weir) # to check if fk_overflow_char also has to be considered if filtered: - query = ( - query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, - ), - ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - ) + query = query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, + ), + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3209,16 +3203,13 @@ def overflow_common(row): query = qgep_session.query(qgep_model.pump) # to check if fk_overflow_char also has to be considered if filtered: - query = ( - query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, - ), - ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - ) + query = query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, + ), + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3265,16 +3256,13 @@ def overflow_common(row): query = qgep_session.query(qgep_model.leapingweir) # to check if fk_overflow_char also has to be considered if filtered: - query = ( - query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, - qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, - ), - ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - ) + query = query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_wastewater_node, + qgep_model.wastewater_node.obj_id == qgep_model.prank_weir.fk_overflow_to, + ), + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3317,17 +3305,14 @@ def overflow_common(row): query = qgep_session.query(qgep_model.hydraulic_char_data) # side fk_overflow_char not considered in filter query if filtered: - query = ( - query.join( - qgep_model.wastewater_node, - or_( - qgep_model.wastewater_node.obj_id - == qgep_model.hydraulic_char_data.fk_wastewater_node, - # fk_primary_direction only added with VSA-DSS 2020 - ), - ) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - ) + query = query.join( + qgep_model.wastewater_node, + or_( + qgep_model.wastewater_node.obj_id + == qgep_model.hydraulic_char_data.fk_wastewater_node, + # fk_primary_direction only added with VSA-DSS 2020 + ), + ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From 1b78ab50d387c881d39cbf4817777221bdd18ebb Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 09:07:14 +0100 Subject: [PATCH 152/245] filter pipe_profile - always all --- qgepqwat2ili/qgepdss/export.py | 64 +++++++++++++++++++++++++ qgepqwat2ili/utils/qgep_export_utils.py | 9 ++-- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 4127a08e..ff831f7f 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -338,6 +338,10 @@ def overflow_common(row): logger.info("Exporting QGEP.aquifier -> ABWASSER.grundwasserleiter, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.aquifier) # always export all aquifier + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all aquifier datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.aquifier @@ -374,6 +378,10 @@ def overflow_common(row): logger.info("Exporting QGEP.river -> ABWASSER.fliessgewaesser, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.river) # always export all river + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all river datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.river @@ -408,6 +416,10 @@ def overflow_common(row): logger.info("Exporting QGEP.lake -> ABWASSER.see, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.lake) # always export all lake + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all lake datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.lake @@ -444,6 +456,10 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.water_course_segment) # always export all water_course_segment + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all water_course_segment datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.water_course_segment @@ -495,6 +511,10 @@ def overflow_common(row): logger.info("Exporting QGEP.water_catchment -> ABWASSER.wasserfassung, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.water_catchment) # always export all water_catchment + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all water_catchment datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.water_catchment @@ -530,6 +550,10 @@ def overflow_common(row): logger.info("Exporting QGEP.river_bank -> ABWASSER.ufer, ABWASSER.metaattribute") # always export all river_bank + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all river_bank datasets query = {statement}") query = qgep_session.query(qgep_model.river_bank) for row in query: @@ -571,6 +595,10 @@ def overflow_common(row): logger.info("Exporting QGEP.river_bed -> ABWASSER.gewaessersohle, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.river_bed) # always export all river_bed + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all river_bed datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.river_bed @@ -610,6 +638,10 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.sector_water_body) # always export all sector_water_body + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all sector_water_body datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.sector_water_body @@ -651,6 +683,10 @@ def overflow_common(row): logger.info("Exporting QGEP.administrative_office -> ABWASSER.amt, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.administrative_office) # always export all administrative_office + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all administrative_office datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.administrative_office @@ -686,6 +722,10 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.cooperative) # always export all cooperative + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all cooperative datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.cooperative @@ -719,6 +759,10 @@ def overflow_common(row): logger.info("Exporting QGEP.canton -> ABWASSER.kanton, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.canton) # always export all canton + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all canton datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.canton @@ -755,6 +799,10 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.waste_water_association) # always export all waste_water_association + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all waste_water_association datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.waste_water_association @@ -788,6 +836,10 @@ def overflow_common(row): logger.info("Exporting QGEP.municipality -> ABWASSER.gemeinde, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.municipality) # always export all municipality + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all municipality datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.municipality @@ -829,6 +881,10 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.waste_water_treatment_plant) # always export all waste_water_treatment_plant + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all waste_water_treatment_plant datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.waste_water_treatment_plant @@ -872,6 +928,10 @@ def overflow_common(row): logger.info("Exporting QGEP.private -> ABWASSER.privat, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.private) # always export all private + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all private datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.private @@ -1841,6 +1901,10 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.profile_geometry) # always export all profile_geometry + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all profile_geometry datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.profile_geometry diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 9284c60d..d1569f07 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -430,10 +430,11 @@ def export_special_structure(self): def export_pipe_profile(self): query = self.qgep_session.query(self.qgep_model.pipe_profile) - if self.filtered: - query = query.join(self.qgep_model.reach).filter( - self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) - ) + # always export all pipe_profile + if filtered: + # add sql statement to logger + statement = query.statement + logger.info(f" always export all pipe_profile datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.pipe_profile From a3bb975fbf2e90c9e9b356e615c118beb692e5ce Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 09:12:50 +0100 Subject: [PATCH 153/245] correct filter - filtered not available --- qgepqwat2ili/utils/qgep_export_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d1569f07..a44b71e9 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -431,10 +431,9 @@ def export_special_structure(self): def export_pipe_profile(self): query = self.qgep_session.query(self.qgep_model.pipe_profile) # always export all pipe_profile - if filtered: - # add sql statement to logger - statement = query.statement - logger.info(f" always export all pipe_profile datasets query = {statement}") + # add sql statement to logger + statement = query.statement + logger.info(f" always export all pipe_profile datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.pipe_profile From 8cb6ce9b0bb31edea982c21b8346cbd911176ee9 Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:00:04 +0100 Subject: [PATCH 154/245] add check_fk_in_subsetid to structure_part_common --- qgepqwat2ili/utils/qgep_export_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index a44b71e9..be91d3fd 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -231,7 +231,10 @@ def structure_part_common(self, row): Returns common attributes for structure_part """ return { - "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), + # "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": self.check_fk_in_subsetid( + self.subset_wws_ids, row.fk_wastewater_structure__REL + ), "bemerkung": self.truncate(self.emptystr_to_null(row.remark), 80), "bezeichnung": self.null_to_emptystr(row.identifier), "instandstellung": self.get_vl(row.renovation_demand__REL), From a8f0b165e162c5fbf652678a73c5af2373a171b3 Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:12:31 +0100 Subject: [PATCH 155/245] revert --- qgepqwat2ili/utils/qgep_export_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index be91d3fd..a44b71e9 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -231,10 +231,7 @@ def structure_part_common(self, row): Returns common attributes for structure_part """ return { - # "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), - "abwasserbauwerkref": self.check_fk_in_subsetid( - self.subset_wws_ids, row.fk_wastewater_structure__REL - ), + "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), "bemerkung": self.truncate(self.emptystr_to_null(row.remark), 80), "bezeichnung": self.null_to_emptystr(row.identifier), "instandstellung": self.get_vl(row.renovation_demand__REL), From 0d316f34cf1f8fd47d725d78411ac0c5cd433ee1 Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:20:52 +0100 Subject: [PATCH 156/245] adapt filter of subclasses of structure_part --- qgepqwat2ili/utils/qgep_export_utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index a44b71e9..ac834585 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -690,11 +690,17 @@ def export_dryweather_flume(self): def export_cover(self): query = self.qgep_session.query(self.qgep_model.cover) + # if self.filtered: + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) + # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: query = ( query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) - .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) for row in query: # AVAILABLE FIELDS IN QGEP.cover From 71dc5209b847fb21942a3119410256372202e88b Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:26:32 +0100 Subject: [PATCH 157/245] continued --- qgepqwat2ili/qgepdss/export.py | 77 ++++++++++++++++++++----- qgepqwat2ili/utils/qgep_export_utils.py | 40 ++++++++++--- 2 files changed, 93 insertions(+), 24 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index ff831f7f..16b92cdc 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2180,15 +2180,24 @@ def overflow_common(row): "Exporting QGEP.electric_equipment -> ABWASSER.elektrischeeinrichtung, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.electric_equipment) - if filtered: + # if filtered: + # query = ( + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # filtering only on wastewater_structures that are in subset_wws_ids query = ( query.join( qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -2231,9 +2240,14 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.electromechanical_equipment) if filtered: - - # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( - # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # query = ( + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) query = ( query.join( @@ -2241,8 +2255,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3434,14 +3447,22 @@ def overflow_common(row): query = qgep_session.query(qgep_model.backflow_prevention) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: + # query = ( + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) query = ( query.join( qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3486,14 +3507,22 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.solids_retention) if filtered: + # query = ( + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) query = ( query.join( qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3536,14 +3565,22 @@ def overflow_common(row): logger.info("Exporting QGEP.tank_cleaning -> ABWASSER.beckenreinigung, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.tank_cleaning) if filtered: + # query = ( + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) query = ( query.join( qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3587,14 +3624,22 @@ def overflow_common(row): query = qgep_session.query(qgep_model.tank_emptying) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: + query = ( + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) query = ( query.join( qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .join(qgep_model.wastewater_networkelement) - .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index ac834585..8192a89c 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -573,11 +573,17 @@ def export_reach(self): def export_dryweather_downspout(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) + # if self.filtered: + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) + # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: query = ( query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) - .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout @@ -612,11 +618,17 @@ def export_dryweather_downspout(self): def export_access_aid(self): query = self.qgep_session.query(self.qgep_model.access_aid) + # if self.filtered: + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) + # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: query = ( query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) - .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) for row in query: # AVAILABLE FIELDS IN QGEP.access_aid @@ -651,11 +663,17 @@ def export_access_aid(self): def export_dryweather_flume(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) + # if self.filtered: + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) + # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: query = ( query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) - .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_flume @@ -744,11 +762,17 @@ def export_cover(self): def export_benching(self): query = self.qgep_session.query(self.qgep_model.benching) + # if self.filtered: + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) + # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: query = ( query.join(self.qgep_model.wastewater_structure) - .join(self.qgep_model.wastewater_networkelement) - .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) for row in query: # AVAILABLE FIELDS IN QGEP.benching From f62b82c1093ce6a8ba58b21be72541a3b916e8ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:26:47 +0000 Subject: [PATCH 158/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 77 ++++++++++++------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 8192a89c..b19ddb41 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -574,16 +574,15 @@ def export_reach(self): def export_dryweather_downspout(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) # if self.filtered: - # query = ( - # query.join(self.qgep_model.wastewater_structure) - # .join(self.qgep_model.wastewater_networkelement) - # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # ) + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = ( - query.join(self.qgep_model.wastewater_structure) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout @@ -619,16 +618,15 @@ def export_dryweather_downspout(self): def export_access_aid(self): query = self.qgep_session.query(self.qgep_model.access_aid) # if self.filtered: - # query = ( - # query.join(self.qgep_model.wastewater_structure) - # .join(self.qgep_model.wastewater_networkelement) - # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # ) + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = ( - query.join(self.qgep_model.wastewater_structure) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) for row in query: # AVAILABLE FIELDS IN QGEP.access_aid @@ -664,16 +662,15 @@ def export_access_aid(self): def export_dryweather_flume(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) # if self.filtered: - # query = ( - # query.join(self.qgep_model.wastewater_structure) - # .join(self.qgep_model.wastewater_networkelement) - # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # ) + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = ( - query.join(self.qgep_model.wastewater_structure) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_flume @@ -709,16 +706,15 @@ def export_dryweather_flume(self): def export_cover(self): query = self.qgep_session.query(self.qgep_model.cover) # if self.filtered: - # query = ( - # query.join(self.qgep_model.wastewater_structure) - # .join(self.qgep_model.wastewater_networkelement) - # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # ) + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = ( - query.join(self.qgep_model.wastewater_structure) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) for row in query: # AVAILABLE FIELDS IN QGEP.cover @@ -762,17 +758,16 @@ def export_cover(self): def export_benching(self): query = self.qgep_session.query(self.qgep_model.benching) - # if self.filtered: - # query = ( - # query.join(self.qgep_model.wastewater_structure) - # .join(self.qgep_model.wastewater_networkelement) - # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # ) + # if self.filtered: + # query = ( + # query.join(self.qgep_model.wastewater_structure) + # .join(self.qgep_model.wastewater_networkelement) + # .filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = ( - query.join(self.qgep_model.wastewater_structure) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) for row in query: # AVAILABLE FIELDS IN QGEP.benching From 1b44c8e0f56a1e0cddd433e56498bda23e6a5bbf Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:28:32 +0100 Subject: [PATCH 159/245] missing if filtered --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 16b92cdc..153f9005 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2180,7 +2180,7 @@ def overflow_common(row): "Exporting QGEP.electric_equipment -> ABWASSER.elektrischeeinrichtung, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.electric_equipment) - # if filtered: + if filtered: # query = ( # query.join( # qgep_model.wastewater_structure, From 4ef7d807038475ca630fdad07f6cab6baf541cdb Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:29:49 +0100 Subject: [PATCH 160/245] correction --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 153f9005..960a5877 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -3624,7 +3624,7 @@ def overflow_common(row): query = qgep_session.query(qgep_model.tank_emptying) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: - query = ( + # query = ( # query.join( # qgep_model.wastewater_structure, # qgep_model.structure_part.fk_wastewater_structure From f594e68d4616fc06573393e3d34cbe2ca3bd6767 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:30:03 +0000 Subject: [PATCH 161/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 162 +++++++++++++++------------------ 1 file changed, 72 insertions(+), 90 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 960a5877..f7ed8400 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2182,23 +2182,20 @@ def overflow_common(row): query = qgep_session.query(qgep_model.electric_equipment) if filtered: # query = ( - # query.join( - # qgep_model.wastewater_structure, - # qgep_model.structure_part.fk_wastewater_structure - # == qgep_model.wastewater_structure.obj_id, - # ) - # .join(qgep_model.wastewater_networkelement) - # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, + # ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) # filtering only on wastewater_structures that are in subset_wws_ids - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -2241,22 +2238,19 @@ def overflow_common(row): query = qgep_session.query(qgep_model.electromechanical_equipment) if filtered: # query = ( - # query.join( - # qgep_model.wastewater_structure, - # qgep_model.structure_part.fk_wastewater_structure - # == qgep_model.wastewater_structure.obj_id, - # ) - # .join(qgep_model.wastewater_networkelement) - # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3448,22 +3442,19 @@ def overflow_common(row): # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: # query = ( - # query.join( - # qgep_model.wastewater_structure, - # qgep_model.structure_part.fk_wastewater_structure - # == qgep_model.wastewater_structure.obj_id, - # ) - # .join(qgep_model.wastewater_networkelement) - # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3508,22 +3499,19 @@ def overflow_common(row): query = qgep_session.query(qgep_model.solids_retention) if filtered: # query = ( - # query.join( - # qgep_model.wastewater_structure, - # qgep_model.structure_part.fk_wastewater_structure - # == qgep_model.wastewater_structure.obj_id, - # ) - # .join(qgep_model.wastewater_networkelement) - # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3566,22 +3554,19 @@ def overflow_common(row): query = qgep_session.query(qgep_model.tank_cleaning) if filtered: # query = ( - # query.join( - # qgep_model.wastewater_structure, - # qgep_model.structure_part.fk_wastewater_structure - # == qgep_model.wastewater_structure.obj_id, - # ) - # .join(qgep_model.wastewater_networkelement) - # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3625,22 +3610,19 @@ def overflow_common(row): # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: # query = ( - # query.join( - # qgep_model.wastewater_structure, - # qgep_model.structure_part.fk_wastewater_structure - # == qgep_model.wastewater_structure.obj_id, - # ) - # .join(qgep_model.wastewater_networkelement) - # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query.join( + # qgep_model.wastewater_structure, + # qgep_model.structure_part.fk_wastewater_structure + # == qgep_model.wastewater_structure.obj_id, # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + # .join(qgep_model.wastewater_networkelement) + # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From 17ec942ba3b04f5aa4723ed297bfe32b58f4e558 Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:31:33 +0100 Subject: [PATCH 162/245] correct self. --- qgepqwat2ili/qgepdss/export.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 960a5877..dcf0b8d5 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2197,7 +2197,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -2255,7 +2255,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3462,7 +3462,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3522,7 +3522,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3580,7 +3580,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -3639,7 +3639,7 @@ def overflow_common(row): qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, ) - .filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement From e0bf95147cd1b98f6dd6347355b09f113e484608 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:36:47 +0000 Subject: [PATCH 163/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 78 +++++++++++++--------------------- 1 file changed, 30 insertions(+), 48 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 1f4c5372..9636c011 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2191,14 +2191,11 @@ def overflow_common(row): # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) # filtering only on wastewater_structures that are in subset_wws_ids - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -2249,14 +2246,11 @@ def overflow_common(row): # .join(qgep_model.wastewater_networkelement) # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3456,14 +3450,11 @@ def overflow_common(row): # .join(qgep_model.wastewater_networkelement) # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3516,14 +3507,11 @@ def overflow_common(row): # .join(qgep_model.wastewater_networkelement) # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3574,14 +3562,11 @@ def overflow_common(row): # .join(qgep_model.wastewater_networkelement) # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3633,14 +3618,11 @@ def overflow_common(row): # .join(qgep_model.wastewater_networkelement) # .filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) # ) - query = ( - query.join( - qgep_model.wastewater_structure, - qgep_model.structure_part.fk_wastewater_structure - == qgep_model.wastewater_structure.obj_id, - ) - .filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) - ) + query = query.join( + qgep_model.wastewater_structure, + qgep_model.structure_part.fk_wastewater_structure + == qgep_model.wastewater_structure.obj_id, + ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From d921128fc0fbf2f3e7ce23fa1f4246232105886a Mon Sep 17 00:00:00 2001 From: SJiB Date: Sat, 14 Dec 2024 15:38:29 +0100 Subject: [PATCH 164/245] remove self. --- qgepqwat2ili/qgepdss/export.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 9636c011..8b353dd0 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2195,7 +2195,7 @@ def overflow_common(row): qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, - ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + ).filter(qgep_model.wastewater_structure.obj_id.in_(subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -2250,7 +2250,7 @@ def overflow_common(row): qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, - ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + ).filter(qgep_model.wastewater_structure.obj_id.in_(subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3454,7 +3454,7 @@ def overflow_common(row): qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, - ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + ).filter(qgep_model.wastewater_structure.obj_id.in_(subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3511,7 +3511,7 @@ def overflow_common(row): qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, - ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + ).filter(qgep_model.wastewater_structure.obj_id.in_(subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3566,7 +3566,7 @@ def overflow_common(row): qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, - ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + ).filter(qgep_model.wastewater_structure.obj_id.in_(subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -3622,7 +3622,7 @@ def overflow_common(row): qgep_model.wastewater_structure, qgep_model.structure_part.fk_wastewater_structure == qgep_model.wastewater_structure.obj_id, - ).filter(qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + ).filter(qgep_model.wastewater_structure.obj_id.in_(subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From c6adb376e8cd72e8628bb9ccdac377e522704078 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 10:37:10 +0100 Subject: [PATCH 165/245] set flag ws_off_sia405abwasse --- qgepqwat2ili/qgepsia405/export.py | 12 ++++++++++++ qgepqwat2ili/utils/qgep_export_utils.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 10ee6507..9d4ea37b 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -51,6 +51,8 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske current_basket = basket_utils.basket_topic_sia405_abwasser + # 0. Initialize ws_off_sia405abwasser + ws_off_sia405abwasser = False # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] @@ -116,6 +118,11 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ws_off_sia405abwasser_list = None ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + # set flag if there are wwtp_structures + ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + logger.info( + f"ws_off_sia405abwasser = {ws_off_sia405abwasser}", + ) # 10. Show ws_off_sia405abwasser_list logger.info( f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", @@ -132,6 +139,11 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ws_off_sia405abwasser_list = None ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + # set flag if there are wwtp_structures + ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + logger.info( + f"ws_off_sia405abwasser (non filtered) = {ws_off_sia405abwasser}", + ) # 21. Show ws_off_sia405abwasser_list logger.info( f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index b19ddb41..b69e9ae5 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -21,6 +21,7 @@ def __init__( filtered, subset_ids, subset_wws_ids, + ws_off_sia405abwasser, ): self.tid_maker = tid_maker self.current_basket = current_basket @@ -32,6 +33,7 @@ def __init__( self.filtered = filtered self.subset_ids = subset_ids self.subset_wws_ids = subset_wws_ids + self.ws_off_sia405abwasser = ws_off_sia405abwasser def get_tid(self, relation): """ From 9e5aaf4be82a8465f7259cb6f3c13f1da33bc08e Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 10:45:53 +0100 Subject: [PATCH 166/245] missing attribut in call --- qgepqwat2ili/qgepsia405/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9d4ea37b..ed9b3920 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -178,6 +178,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske filtered=filtered, subset_ids=subset_ids, subset_wws_ids=subset_wws_ids, + ws_off_sia405abwasser=subset_wws_ids, ) # ADAPTED FROM 052a_sia405_abwasser_2015_2_d_interlisexport2.sql From d7007795d810b795f64acf1356ac89c1ce693d21 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 11:02:56 +0100 Subject: [PATCH 167/245] correct --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index ed9b3920..4dc43ab2 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -178,7 +178,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske filtered=filtered, subset_ids=subset_ids, subset_wws_ids=subset_wws_ids, - ws_off_sia405abwasser=subset_wws_ids, + ws_off_sia405abwasser=ws_off_sia405abwasser, ) # ADAPTED FROM 052a_sia405_abwasser_2015_2_d_interlisexport2.sql From 598f1b16b6b2672fc5c4925014013fd92940826f Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 11:11:13 +0100 Subject: [PATCH 168/245] add export_dryweather_downspout_ws_off_sia405abwasser and export_dryweather_flume_ws_off_sia405abwasser --- qgepqwat2ili/qgepsia405/export.py | 50 ++++++++++---- qgepqwat2ili/utils/qgep_export_utils.py | 88 +++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 14 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 4dc43ab2..54dfc73c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -342,24 +342,46 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() - logger.info( - "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" - ) - qgep_export_utils.export_dryweather_downspout() + if ws_off_sia405abwasser: - logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") - qgep_export_utils.export_access_aid() + logger.info( + "Exporting QGEP.dryweather_downspout (ws_off_sia405abwasser) -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_downspout_ws_off_sia405abwasser() - logger.info( - "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" - ) - qgep_export_utils.export_dryweather_flume() + logger.info("Exporting QGEP.access_aid (ws_off_sia405abwasser) -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") + qgep_export_utils.export_access_aid_ws_off_sia405abwasser() + + logger.info( + "Exporting QGEP.dryweather_flume (ws_off_sia405abwasser)-> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_flume_ws_off_sia405abwasser() + + logger.info("Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute") + qgep_export_utils.export_coverws_off_sia405abwasser() + + logger.info("Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute") + qgep_export_utils.export_benchingws_off_sia405abwasser() + + else: + logger.info( + "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_downspout() + + logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") + qgep_export_utils.export_access_aid() + + logger.info( + "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_flume() - logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") - qgep_export_utils.export_cover() + logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") + qgep_export_utils.export_cover() - logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") - qgep_export_utils.export_benching() + logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") + qgep_export_utils.export_benching() # Labels # Note: these are extracted from the optional labels file (not exported from the QGEP database) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index b69e9ae5..d42682a6 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -239,6 +239,19 @@ def structure_part_common(self, row): "instandstellung": self.get_vl(row.renovation_demand__REL), } + def structure_part_common_check_fk_in_subset(self, row): + """ + Returns common attributes for structure_part + """ + return { + "abwasserbauwerkref": self.check_fk_in_subsetid( + self.subset_wws_ids, row.fk_wastewater_structure__REL + ), + "bemerkung": self.truncate(self.emptystr_to_null(row.remark), 80), + "bezeichnung": self.null_to_emptystr(row.identifier), + "instandstellung": self.get_vl(row.renovation_demand__REL), + } + def textpos_common(self, row, t_type, geojson_crs_def): """ Returns common attributes for textpos @@ -617,6 +630,44 @@ def export_dryweather_downspout(self): logger.info("done") self.abwasser_session.flush() + def export_dryweather_downspout_ws_off_sia405abwasser(self): + query = self.qgep_session.query(self.qgep_model.dryweather_downspout) + # if ws_off_sia405abwasser always filter out with subset_wws_ids + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) + + for row in query: + # AVAILABLE FIELDS IN QGEP.dryweather_downspout + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_downspout --- + # diameter, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, renovation_demand__REL + + trockenwetterfallrohr = self.abwasser_model.trockenwetterfallrohr( + # FIELDS TO MAP TO ABWASSER.trockenwetterfallrohr + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "trockenwetterfallrohr"), + # --- bauwerksteil --- + **self.structure_part_common(row), + # --- trockenwetterfallrohr --- + durchmesser=row.diameter, + ) + self.abwasser_session.add(trockenwetterfallrohr) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + def export_access_aid(self): query = self.qgep_session.query(self.qgep_model.access_aid) # if self.filtered: @@ -705,6 +756,43 @@ def export_dryweather_flume(self): logger.info("done") self.abwasser_session.flush() + def export_dryweather_flume_ws_off_sia405abwasser(self): + query = self.qgep_session.query(self.qgep_model.dryweather_flume) + # if ws_off_sia405abwasser always filter out with subset_wws_ids + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) + for row in query: + # AVAILABLE FIELDS IN QGEP.dryweather_flume + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_flume --- + # material, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, renovation_demand__REL + + trockenwetterrinne = self.abwasser_model.trockenwetterrinne( + # FIELDS TO MAP TO ABWASSER.trockenwetterrinne + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "trockenwetterrinne"), + # --- bauwerksteil --- + **self.structure_part_common(row), + # --- trockenwetterrinne --- + material=self.get_vl(row.material__REL), + ) + self.abwasser_session.add(trockenwetterrinne) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + def export_cover(self): query = self.qgep_session.query(self.qgep_model.cover) # if self.filtered: From fc20fc0711543e1f31e9b1bbf29a4e86f34eed84 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:11:30 +0000 Subject: [PATCH 169/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 12 +++++++++--- qgepqwat2ili/utils/qgep_export_utils.py | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 54dfc73c..558e3e25 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -349,7 +349,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ) qgep_export_utils.export_dryweather_downspout_ws_off_sia405abwasser() - logger.info("Exporting QGEP.access_aid (ws_off_sia405abwasser) -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.access_aid (ws_off_sia405abwasser) -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute" + ) qgep_export_utils.export_access_aid_ws_off_sia405abwasser() logger.info( @@ -357,10 +359,14 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ) qgep_export_utils.export_dryweather_flume_ws_off_sia405abwasser() - logger.info("Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute" + ) qgep_export_utils.export_coverws_off_sia405abwasser() - logger.info("Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute" + ) qgep_export_utils.export_benchingws_off_sia405abwasser() else: diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d42682a6..5365d6b9 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -634,8 +634,8 @@ def export_dryweather_downspout_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join(self.qgep_model.wastewater_structure).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout @@ -760,8 +760,8 @@ def export_dryweather_flume_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join(self.qgep_model.wastewater_structure).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_flume From f5370b66be6f5d3699159309d808281180f3fdcb Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 11:47:56 +0100 Subject: [PATCH 170/245] add also for cover, benching and access_aid --- qgepqwat2ili/qgepsia405/export.py | 6 +- qgepqwat2ili/utils/qgep_export_utils.py | 120 ++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 54dfc73c..efbfcecf 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -343,7 +343,6 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske qgep_export_utils.export_reach() if ws_off_sia405abwasser: - logger.info( "Exporting QGEP.dryweather_downspout (ws_off_sia405abwasser) -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" ) @@ -351,17 +350,16 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("Exporting QGEP.access_aid (ws_off_sia405abwasser) -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") qgep_export_utils.export_access_aid_ws_off_sia405abwasser() - logger.info( "Exporting QGEP.dryweather_flume (ws_off_sia405abwasser)-> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" ) qgep_export_utils.export_dryweather_flume_ws_off_sia405abwasser() logger.info("Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute") - qgep_export_utils.export_coverws_off_sia405abwasser() + qgep_export_utils.export_cover_ws_off_sia405abwasser() logger.info("Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute") - qgep_export_utils.export_benchingws_off_sia405abwasser() + qgep_export_utils.export_benching_ws_off_sia405abwasser() else: logger.info( diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d42682a6..0f69c741 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -712,6 +712,43 @@ def export_access_aid(self): logger.info("done") self.abwasser_session.flush() + def export_access_aid_ws_off_sia405abwasser(self): + query = self.qgep_session.query(self.qgep_model.access_aid) + # if ws_off_sia405abwasser always filter out with subset_wws_ids + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) + for row in query: + # AVAILABLE FIELDS IN QGEP.access_aid + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- access_aid --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + einstiegshilfe = self.abwasser_model.einstiegshilfe( + # FIELDS TO MAP TO ABWASSER.einstiegshilfe + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "einstiegshilfe"), + # --- bauwerksteil --- + **self.structure_part_common(row), + # --- einstiegshilfe --- + art=self.get_vl(row.kind__REL), + ) + self.abwasser_session.add(einstiegshilfe) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + def export_dryweather_flume(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) # if self.filtered: @@ -846,6 +883,53 @@ def export_cover(self): logger.info("done") self.abwasser_session.flush() + def export_cover_ws_off_sia405abwasser(self): + query = self.qgep_session.query(self.qgep_model.cover) + # if ws_off_sia405abwasser always filter out with subset_wws_ids + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) + for row in query: + # AVAILABLE FIELDS IN QGEP.cover + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- cover --- + # brand, cover_shape, diameter, fastening, level, material, obj_id, positional_accuracy, situation_geometry, sludge_bucket, venting + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id, wastewater_structure__BWREL_fk_main_cover + + # --- _rel_ --- + # cover_shape__REL, fastening__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, positional_accuracy__REL, renovation_demand__REL, sludge_bucket__REL, venting__REL + + deckel = self.abwasser_model.deckel( + # FIELDS TO MAP TO ABWASSER.deckel + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "deckel"), + # --- bauwerksteil --- + **self.structure_part_common(row), + # --- deckel --- + deckelform=self.get_vl(row.cover_shape__REL), + durchmesser=row.diameter, + entlueftung=self.get_vl(row.venting__REL), + fabrikat=row.brand, + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lagegenauigkeit=self.get_vl(row.positional_accuracy__REL), + material=self.get_vl(row.material__REL), + schlammeimer=self.get_vl(row.sludge_bucket__REL), + verschluss=self.get_vl(row.fastening__REL), + ) + self.abwasser_session.add(deckel) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + + def export_benching(self): query = self.qgep_session.query(self.qgep_model.benching) # if self.filtered: @@ -890,6 +974,42 @@ def export_benching(self): logger.info("done") self.abwasser_session.flush() + def export_benching_ws_off_sia405abwasser(self): + query = self.qgep_session.query(self.qgep_model.benching) + # if ws_off_sia405abwasser always filter out with subset_wws_ids + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) + for row in query: + # AVAILABLE FIELDS IN QGEP.benching + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- benching --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + bankett = self.abwasser_model.bankett( + # FIELDS TO MAP TO ABWASSER.bankett + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "bankett"), + # --- bauwerksteil --- + **self.structure_part_common(row), + # --- bankett --- + art=self.get_vl(row.kind__REL), + ) + self.abwasser_session.add(bankett) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() # end class QgepExportUtils From 500e902910fdd43196908ff91e83dc1a24c97a8d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:49:27 +0000 Subject: [PATCH 171/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 8 ++++++-- qgepqwat2ili/utils/qgep_export_utils.py | 14 +++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index dba6803f..9425d379 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -357,10 +357,14 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ) qgep_export_utils.export_dryweather_flume_ws_off_sia405abwasser() - logger.info("Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute" + ) qgep_export_utils.export_cover_ws_off_sia405abwasser() - logger.info("Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute" + ) qgep_export_utils.export_benching_ws_off_sia405abwasser() else: diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 43ade57a..6bfc6735 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -716,8 +716,8 @@ def export_access_aid_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.access_aid) # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join(self.qgep_model.wastewater_structure).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) for row in query: # AVAILABLE FIELDS IN QGEP.access_aid @@ -887,8 +887,8 @@ def export_cover_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.cover) # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join(self.qgep_model.wastewater_structure).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) for row in query: # AVAILABLE FIELDS IN QGEP.cover @@ -929,7 +929,6 @@ def export_cover_ws_off_sia405abwasser(self): logger.info("done") self.abwasser_session.flush() - def export_benching(self): query = self.qgep_session.query(self.qgep_model.benching) # if self.filtered: @@ -978,8 +977,8 @@ def export_benching_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.benching) # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join(self.qgep_model.wastewater_structure).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + ) for row in query: # AVAILABLE FIELDS IN QGEP.benching @@ -1011,6 +1010,7 @@ def export_benching_ws_off_sia405abwasser(self): logger.info("done") self.abwasser_session.flush() + # end class QgepExportUtils From 5f39ed0b2c32e5b28205b270b97d9934d11db77b Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 11:50:16 +0100 Subject: [PATCH 172/245] adapt call with ws_off_sia405abwasser --- qgepqwat2ili/qgepdss/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 8b353dd0..3ca6c04d 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -175,6 +175,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e filtered=filtered, subset_ids=subset_ids, subset_wws_ids=subset_wws_ids, + ws_off_sia405abwasser=ws_off_sia405abwasser, ) def organisation_common(row): From 4c985f97cdbcc18ddead3027efbb37845bda2c10 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 11:54:00 +0100 Subject: [PATCH 173/245] define ws_off_sia405abwasser --- qgepqwat2ili/qgepdss/export.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 3ca6c04d..6359e3ef 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -51,6 +51,8 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e current_basket = basket_utils.basket_topic_sia405_abwasser + # 0. Initialize ws_off_sia405abwasser + ws_off_sia405abwasser = False # 1. Filtering - check if selection filtered = selection is not None From 571631646b9f590715f3e85f259b8d82207d322f Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 13:12:55 +0100 Subject: [PATCH 174/245] adapt logger messages --- qgepqwat2ili/qgepsia405/export.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9425d379..bee1efc6 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -68,11 +68,26 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + logger.debug( + "f 5 + 2 adapted_subset_ids: {adapted_subset_ids}", + ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + logger.debug( + "f 5 + 2 + 3 adapted_subset_ids: {adapted_subset_ids}", + ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + logger.debug( + "f 5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", + ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) + logger.debug( + "f 6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + logger.debug( + "f 6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) if adapted_subset_ids_reaches is None: extra_reaches_ids = [] if not adapted_subset_ids_reaches: @@ -112,7 +127,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( - f"subset_wws_ids: {subset_wws_ids}", + f" 8. subset_wws_ids: {subset_wws_ids}", ) # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) ws_off_sia405abwasser_list = None @@ -121,16 +136,16 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # set flag if there are wwtp_structures ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None logger.info( - f"ws_off_sia405abwasser = {ws_off_sia405abwasser}", + f"9. ws_off_sia405abwasser = {ws_off_sia405abwasser}", ) # 10. Show ws_off_sia405abwasser_list logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + f"10. ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + f"11. subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", ) # also if not filtered we have to take out references to wwtp_structures @@ -142,22 +157,22 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # set flag if there are wwtp_structures ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None logger.info( - f"ws_off_sia405abwasser (non filtered) = {ws_off_sia405abwasser}", + f"20. ws_off_sia405abwasser (non filtered) = {ws_off_sia405abwasser}", ) # 21. Show ws_off_sia405abwasser_list logger.info( - f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", + f"21. ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", ) # 22. Get list of all wastewater_structures subset_wws_ids = get_ws_ids("wastewater_structure") logger.info( - f"subset_wws_ids (non filtered) : {subset_wws_ids}", + f"22. subset_wws_ids (non filtered) : {subset_wws_ids}", ) # 23. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", + f"23. subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", ) # Orientation From 1d2df79b1d0cad3793a6a95f35e612d5aff962be Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 13:21:37 +0100 Subject: [PATCH 175/245] correct f" --- qgepqwat2ili/qgepsia405/export.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index bee1efc6..2d5c01d0 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -69,24 +69,24 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) logger.debug( - "f 5 + 2 adapted_subset_ids: {adapted_subset_ids}", + f"5 + 2 adapted_subset_ids: {adapted_subset_ids}", ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) logger.debug( - "f 5 + 2 + 3 adapted_subset_ids: {adapted_subset_ids}", + f"5 + 2 + 3 adapted_subset_ids: {adapted_subset_ids}", ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) logger.debug( - "f 5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", + f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) logger.debug( - "f 6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) logger.debug( - "f 6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) if adapted_subset_ids_reaches is None: extra_reaches_ids = [] @@ -96,7 +96,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ) else: logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + f"adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) # https://www.geeksforgeeks.org/python-difference-two-lists/ # First convert lists to sets @@ -127,7 +127,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( - f" 8. subset_wws_ids: {subset_wws_ids}", + f"8. subset_wws_ids: {subset_wws_ids}", ) # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) ws_off_sia405abwasser_list = None From c06434ecd52b29288fbcbbdcdd2b80e9f8384d38 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 13:25:50 +0100 Subject: [PATCH 176/245] initialize --- qgepqwat2ili/qgepdss/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 6359e3ef..77aa0ffc 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -80,6 +80,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = [] adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) if adapted_subset_ids_reaches is None: extra_reaches_ids = [] From 7f9c39d07b3a3aa1c20e6fc98f31d14566647682 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 13:27:44 +0100 Subject: [PATCH 177/245] dito sia405 --- qgepqwat2ili/qgepsia405/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 2d5c01d0..63ff2761 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -84,6 +84,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.debug( f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) + adapted_subset_ids_reaches = [] adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) logger.debug( f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", From 700687b308198d265050de1a960e881165be3a2e Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 13:40:31 +0100 Subject: [PATCH 178/245] correct remove_from_selection and add_to_selection --- qgepqwat2ili/qgepsia405/export.py | 3 ++- qgepqwat2ili/utils/qgep_export_utils.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 63ff2761..61f846b2 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -80,9 +80,10 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = [] subset_ids_reaches = filter_reaches(subset_ids) logger.debug( - f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + f"6. subset_ids_reaches: {subset_ids_reaches}", ) adapted_subset_ids_reaches = [] adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 6bfc6735..e805c010 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1395,8 +1395,10 @@ def remove_from_selection(selected_ids, remove_ids): Remove ids from selected_ids if they are in selected_ids """ - if remove_ids is None: + if selected_ids is None: return None + if remove_ids is None: + return selected_ids else: for list_item in remove_ids: # selected_ids = selected_ids.remove(list_item) @@ -1414,8 +1416,10 @@ def add_to_selection(selected_ids, add_ids): """ Append ids to selected_ids """ - if add_ids is None: + if selected_ids is None: return None + if add_ids is None: + return selected_ids else: if selected_ids is None: selected_ids = [] From 4a6a45ae5400f824a259089ef04984c4ffe76f46 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 14:16:45 +0100 Subject: [PATCH 179/245] update export dss --- qgepqwat2ili/qgepdss/export.py | 43 ++++++++++++++++++++++++++----- qgepqwat2ili/qgepsia405/export.py | 3 +++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 77aa0ffc..3da1950f 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -76,12 +76,28 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + logger.debug( + f"5 + 2 adapted_subset_ids: {adapted_subset_ids}", + ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + logger.debug( + f"5 + 2 + 3 adapted_subset_ids: {adapted_subset_ids}", + ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + logger.debug( + f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", + ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = [] subset_ids_reaches = filter_reaches(subset_ids) + logger.debug( + f"6. subset_ids_reaches: {subset_ids_reaches}", + ) adapted_subset_ids_reaches = [] adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + logger.debug( + f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) if adapted_subset_ids_reaches is None: extra_reaches_ids = [] if not adapted_subset_ids_reaches: @@ -90,7 +106,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e ) else: logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + f"adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) # https://www.geeksforgeeks.org/python-difference-two-lists/ # First convert lists to sets @@ -121,20 +137,25 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( - f"subset_wws_ids: {subset_wws_ids}", + f"8. subset_wws_ids: {subset_wws_ids}", ) # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) # ws_off_sia405abwasser_list = None # ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + # set flag if there are wwtp_structures + # ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + # logger.info( + # f"9. ws_off_sia405abwasser = {ws_off_sia405abwasser}", + # ) # 10. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + # f"10. ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # f"11. subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", # ) # also if not filtered we have to take out references to wwtp_structures @@ -143,20 +164,25 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # ws_off_sia405abwasser_list = None # ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + # set flag if there are wwtp_structures + ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + logger.info( + f"20. ws_off_sia405abwasser (non filtered) = {ws_off_sia405abwasser}", + ) # 21. Show ws_off_sia405abwasser_list # logger.info( - # f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", + # f"21. ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", # ) # 22. Get list of all wastewater_structures # subset_wws_ids = get_ws_ids("wastewater_structure") # logger.info( - # f"subset_wws_ids (non filtered) : {subset_wws_ids}", + # f"22. subset_wws_ids (non filtered) : {subset_wws_ids}", # ) # 23. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", + # f"23. subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", # ) logger.debug("Handling of wwtp_structures not needed with VSA-DSS") @@ -1052,6 +1078,9 @@ def overflow_common(row): # --- _rel_ --- # accessibility__REL, defects__REL, emergency_spillway__REL, financing__REL, fk_aquifier__REL, fk_dataowner__REL, fk_main_cover__REL, fk_main_wastewater_node__REL, fk_operator__REL, fk_owner__REL, fk_provider__REL, kind__REL, labeling__REL, renovation_necessity__REL, rv_construction_type__REL, seepage_utilization__REL, status__REL, structure_condition__REL, vehicle_access__REL, watertightness__REL + logger.info( + "QGEP field infiltration_installation.upper_elevation is part of 3D extension. It will be ignored." + ) versickerungsanlage = abwasser_model.versickerungsanlage( # FIELDS TO MAP TO ABWASSER.versickerungsanlage # --- baseclass --- diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 61f846b2..22f67c5c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -321,6 +321,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske query = qgep_session.query(qgep_model.wastewater_node) if filtered: query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.wastewater_node From 5d0d3425bf40c06b2dc7594e8f6b3577e6185e5b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:18:04 +0000 Subject: [PATCH 180/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 3da1950f..a3583fba 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -146,16 +146,16 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # set flag if there are wwtp_structures # ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None # logger.info( - # f"9. ws_off_sia405abwasser = {ws_off_sia405abwasser}", + # f"9. ws_off_sia405abwasser = {ws_off_sia405abwasser}", # ) # 10. Show ws_off_sia405abwasser_list # logger.info( - # f"10. ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + # f"10. ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", # ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"11. subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + # f"11. subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", # ) # also if not filtered we have to take out references to wwtp_structures @@ -171,18 +171,18 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e ) # 21. Show ws_off_sia405abwasser_list # logger.info( - # f"21. ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", + # f"21. ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", # ) # 22. Get list of all wastewater_structures # subset_wws_ids = get_ws_ids("wastewater_structure") # logger.info( - # f"22. subset_wws_ids (non filtered) : {subset_wws_ids}", + # f"22. subset_wws_ids (non filtered) : {subset_wws_ids}", # ) # 23. take out ws_off_sia405abwasser_list from subset_wws_ids # subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) # logger.info( - # f"23. subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", + # f"23. subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", # ) logger.debug("Handling of wwtp_structures not needed with VSA-DSS") From ccb1b047bbf34f4a5b70506ea5465ec9bf591a16 Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 14:22:38 +0100 Subject: [PATCH 181/245] update export kek --- qgepqwat2ili/qgep/export.py | 101 ++++++++++++++++++++++++++------- qgepqwat2ili/qgepdss/export.py | 1 + 2 files changed, 81 insertions(+), 21 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index f86c3c76..cf838817 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -52,6 +52,8 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e current_basket = basket_utils.basket_topic_sia405_abwasser + # 0. Initialize ws_off_sia405abwasser + ws_off_sia405abwasser = False # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] @@ -67,11 +69,28 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids adapted_subset_ids = [] adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + logger.debug( + f"5 + 2 adapted_subset_ids: {adapted_subset_ids}", + ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + logger.debug( + f"5 + 2 + 3 adapted_subset_ids: {adapted_subset_ids}", + ) adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + logger.debug( + f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", + ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = [] subset_ids_reaches = filter_reaches(subset_ids) + logger.debug( + f"6. subset_ids_reaches: {subset_ids_reaches}", + ) + adapted_subset_ids_reaches = [] adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) + logger.debug( + f"6. adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + ) if adapted_subset_ids_reaches is None: extra_reaches_ids = [] if not adapted_subset_ids_reaches: @@ -80,7 +99,7 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e ) else: logger.debug( - "f adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", + f"adapted_subset_ids_reaches: {adapted_subset_ids_reaches}", ) # https://www.geeksforgeeks.org/python-difference-two-lists/ # First convert lists to sets @@ -111,20 +130,25 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e # 8. get all id's of connected wastewater_structures subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) logger.info( - f"subset_wws_ids: {subset_wws_ids}", + f"8. subset_wws_ids: {subset_wws_ids}", ) # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) ws_off_sia405abwasser_list = None ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + # set flag if there are wwtp_structures + ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + logger.info( + f"9. ws_off_sia405abwasser = {ws_off_sia405abwasser}", + ) # 10. Show ws_off_sia405abwasser_list logger.info( - f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + f"10. ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", ) # 11. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + f"11. subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", ) # also if not filtered we have to take out references to wwtp_structures @@ -133,20 +157,25 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e ws_off_sia405abwasser_list = None ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") + # set flag if there are wwtp_structures + ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + logger.info( + f"20. ws_off_sia405abwasser (non filtered) = {ws_off_sia405abwasser}", + ) # 21. Show ws_off_sia405abwasser_list logger.info( - f"ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", + f"21. ws_off_sia405abwasser_list (non filtered) : {ws_off_sia405abwasser_list}", ) # 22. Get list of all wastewater_structures subset_wws_ids = get_ws_ids("wastewater_structure") logger.info( - f"subset_wws_ids (non filtered) : {subset_wws_ids}", + f"22. subset_wws_ids (non filtered) : {subset_wws_ids}", ) # 23. take out ws_off_sia405abwasser_list from subset_wws_ids subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) logger.info( - f"subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", + f"23. subset_ids of all wws minus ws_off_sia405abwasser_list (non filtered): {subset_wws_ids}", ) # Orientation @@ -167,6 +196,7 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e filtered=filtered, subset_ids=subset_ids, subset_wws_ids=subset_wws_ids, + ws_off_sia405abwasser=ws_off_sia405abwasser, ) # ADAPTED FROM 052a_sia405_abwasser_2015_2_d_interlisexport2.sql @@ -292,6 +322,9 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e query = qgep_session.query(qgep_model.wastewater_node) if filtered: query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.wastewater_node @@ -330,24 +363,50 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() - logger.info( - "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" - ) - qgep_export_utils.export_dryweather_downspout() + if ws_off_sia405abwasser: + logger.info( + "Exporting QGEP.dryweather_downspout (ws_off_sia405abwasser) -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_downspout_ws_off_sia405abwasser() + + logger.info( + "Exporting QGEP.access_aid (ws_off_sia405abwasser) -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute" + ) + qgep_export_utils.export_access_aid_ws_off_sia405abwasser() + logger.info( + "Exporting QGEP.dryweather_flume (ws_off_sia405abwasser)-> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_flume_ws_off_sia405abwasser() - logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") - qgep_export_utils.export_access_aid() + logger.info( + "Exporting QGEP.cover (ws_off_sia405abwasser) ( -> ABWASSER.deckel, ABWASSER.metaattribute" + ) + qgep_export_utils.export_cover_ws_off_sia405abwasser() - logger.info( - "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" - ) - qgep_export_utils.export_dryweather_flume() + logger.info( + "Exporting QGEP.benching (ws_off_sia405abwasser) -> ABWASSER.bankett, ABWASSER.metaattribute" + ) + qgep_export_utils.export_benching_ws_off_sia405abwasser() + + else: + logger.info( + "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_downspout() + + logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") + qgep_export_utils.export_access_aid() + + logger.info( + "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" + ) + qgep_export_utils.export_dryweather_flume() - logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") - qgep_export_utils.export_cover() + logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") + qgep_export_utils.export_cover() - logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") - qgep_export_utils.export_benching() + logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") + qgep_export_utils.export_benching() logger.info("Exporting QGEP.examination -> ABWASSER.untersuchung, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.examination) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 3da1950f..3ce54fdb 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -14,6 +14,7 @@ get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, + get_ws_ids, get_ws_selected_ww_networkelements, remove_from_selection, ) From 044ea9862dfc7b50f448707da37f511a0a8f4a4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:23:02 +0000 Subject: [PATCH 182/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 9bd15864..a3583fba 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -14,7 +14,6 @@ get_connected_overflow_to_wn_ids, get_connected_we_from_re, get_connected_we_to_re, - get_ws_ids, get_ws_selected_ww_networkelements, remove_from_selection, ) From 9bd2764983a4ac5216f4063142544ce3db88d7ea Mon Sep 17 00:00:00 2001 From: SJiB Date: Mon, 16 Dec 2024 14:28:42 +0100 Subject: [PATCH 183/245] always false --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index a3583fba..2f988cce 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -164,8 +164,8 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # ws_off_sia405abwasser_list = None # ws_off_sia405abwasser_list = get_ws_ids("wwtp_structure") - # set flag if there are wwtp_structures - ws_off_sia405abwasser = ws_off_sia405abwasser_list is not None + # set flag if there are wwtp_structures - with dss always false + ws_off_sia405abwasser = False logger.info( f"20. ws_off_sia405abwasser (non filtered) = {ws_off_sia405abwasser}", ) From 4b429eaaea48e76e9f8d9db4eeb93ce519fd5914 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 11:00:14 +0100 Subject: [PATCH 184/245] export_reach_check_fk_in_subset added --- qgepqwat2ili/utils/qgep_export_utils.py | 70 ++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index e805c010..8496e53c 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -216,7 +216,17 @@ def wastewater_structure_common(self, row): def wastewater_networkelement_common(self, row): """ - Returns common attributes for wastewater_networkelement + Returns common attributes for wastewater_networkelement - no check_fk_in_subsetid + """ + return { + "abwasserbauwerkref": self.get_tid(row.fk_wastewater_structure__REL), + "bemerkung": self.truncate(self.emptystr_to_null(row.remark), 80), + "bezeichnung": self.null_to_emptystr(row.identifier), + } + + def wastewater_networkelement_common_check_fk_in_subset(self, row): + """ + Returns common attributes for wastewater_networkelement with check_fk_in_subsetid """ return { # added check_fk_in_subsetid with subset_wws_ids (only needed for SIA405 Abwasser export wwtp_structure - but as now in qgep_export_utils done for all export - might slow donw export @@ -586,6 +596,64 @@ def export_reach(self): logger.info("done") self.abwasser_session.flush() + def export_reach_check_fk_in_subset(self): + query = self.qgep_session.query(self.qgep_model.reach) + if self.filtered: + query = query.filter( + self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + ) + for row in query: + # AVAILABLE FIELDS IN QGEP.reach + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- reach --- + # clear_height, coefficient_of_friction, elevation_determination, fk_pipe_profile, fk_reach_point_from, fk_reach_point_to, horizontal_positioning, inside_coating, length_effective, material, obj_id, progression_geometry, reliner_material, reliner_nominal_size, relining_construction, relining_kind, ring_stiffness, slope_building_plan, wall_roughness + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, reach_point__BWREL_fk_wastewater_networkelement, reach_text__BWREL_fk_reach, txt_text__BWREL_fk_reach + + # --- _rel_ --- + # elevation_determination__REL, fk_dataowner__REL, fk_pipe_profile__REL, fk_provider__REL, fk_reach_point_from__REL, fk_reach_point_to__REL, fk_wastewater_structure__REL, horizontal_positioning__REL, inside_coating__REL, material__REL, reliner_material__REL, relining_construction__REL, relining_kind__REL + + # QGEP field reach.elevation_determination has no equivalent in the interlis model. It will be ignored. + + haltung = self.abwasser_model.haltung( + # FIELDS TO MAP TO ABWASSER.haltung + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "haltung"), + # --- abwassernetzelement --- + **self.wastewater_networkelement_common_check_fk_in_subset(row), + # --- haltung --- + # NOT MAPPED : elevation_determination + innenschutz=self.get_vl(row.inside_coating__REL), + laengeeffektiv=row.length_effective, + lagebestimmung=self.get_vl(row.horizontal_positioning__REL), + lichte_hoehe=row.clear_height, + material=self.get_vl(row.material__REL), + nachhaltungspunktref=self.get_tid(row.fk_reach_point_to__REL), + plangefaelle=row.slope_building_plan, # TODO : check, does this need conversion ? + reibungsbeiwert=row.coefficient_of_friction, + reliner_art=self.get_vl(row.relining_kind__REL), + reliner_bautechnik=self.get_vl(row.relining_construction__REL), + reliner_material=self.get_vl(row.reliner_material__REL), + reliner_nennweite=row.reliner_nominal_size, + ringsteifigkeit=row.ring_stiffness, + rohrprofilref=self.get_tid(row.fk_pipe_profile__REL), + verlauf=ST_Force2D(row.progression_geometry), + # -- attribute 3D --- + # verlauf3d=row.progression3d, + vonhaltungspunktref=self.get_tid(row.fk_reach_point_from__REL), + wandrauhigkeit=row.wall_roughness, + ) + self.abwasser_session.add(haltung) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + def export_dryweather_downspout(self): query = self.qgep_session.query(self.qgep_model.dryweather_downspout) # if self.filtered: From 2366f1058eddf9e165b9288e29a3dea312110a92 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 11:05:54 +0100 Subject: [PATCH 185/245] move qgep_model.wastewater_node also to qgep_export_utils --- qgepqwat2ili/qgepdss/export.py | 39 +---------------------- qgepqwat2ili/utils/qgep_export_utils.py | 41 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 2f988cce..5fde976a 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1887,44 +1887,7 @@ def overflow_common(row): logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) - query = qgep_session.query(qgep_model.wastewater_node) - if filtered: - query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # --- wastewater_node --- - - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - abwasserknoten = abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **qgep_export_utils.wastewater_networkelement_common(row), - # --- abwasserknoten --- - hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, - ) - abwasser_session.add(abwasserknoten) - qgep_export_utils.create_metaattributes(row) - print(".", end="") - logger.info("done") - abwasser_session.flush() + qgep_export_utils.export_wastewater_node() logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 8496e53c..5265b560 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -489,6 +489,47 @@ def export_pipe_profile(self): logger.info("done") self.abwasser_session.flush() +def export_wastewater_node(self): + query = self.qgep_session.query(self.qgep_model.wastewater_node) + if filtered: + query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + abwasserknoten = self.abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **self.qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **qgep_export_utils.wastewater_networkelement_common(row), + # --- abwasserknoten --- + hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + self.abwasser_session.add(abwasserknoten) + self.qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + + def export_reach_point(self): query = self.qgep_session.query(self.qgep_model.reach_point) if self.filtered: From c505305d2360680d16d6caa8ce4e6f1f5f5663dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:06:15 +0000 Subject: [PATCH 186/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 5265b560..bb9e2119 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -489,6 +489,7 @@ def export_pipe_profile(self): logger.info("done") self.abwasser_session.flush() + def export_wastewater_node(self): query = self.qgep_session.query(self.qgep_model.wastewater_node) if filtered: @@ -529,7 +530,6 @@ def export_wastewater_node(self): logger.info("done") self.abwasser_session.flush() - def export_reach_point(self): query = self.qgep_session.query(self.qgep_model.reach_point) if self.filtered: From 4969df600aed7a45c3856008ca3c4fb481d4422c Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 11:08:56 +0100 Subject: [PATCH 187/245] correct self. --- qgepqwat2ili/utils/qgep_export_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 5265b560..c4533fee 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -491,8 +491,8 @@ def export_pipe_profile(self): def export_wastewater_node(self): query = self.qgep_session.query(self.qgep_model.wastewater_node) - if filtered: - query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + if self.filtered: + query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -516,9 +516,9 @@ def export_wastewater_node(self): # --- sia405_baseclass --- **self.qgep_export_utils.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- - **qgep_export_utils.wastewater_networkelement_common(row), + **self.qgep_export_utils.wastewater_networkelement_common(row), # --- abwasserknoten --- - hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), lage=ST_Force2D(row.situation_geometry), rueckstaukote=row.backflow_level, sohlenkote=row.bottom_level, From fd67661b2c64b16a5607ebf4045041875db636a9 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 11:14:16 +0100 Subject: [PATCH 188/245] add export_wastewater_node_check_fk_in_subset --- qgepqwat2ili/utils/qgep_export_utils.py | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index c62cae2c..323bb82e 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -530,6 +530,46 @@ def export_wastewater_node(self): logger.info("done") self.abwasser_session.flush() +def export_wastewater_node_check_fk_in_subset(self): + query = self.qgep_session.query(self.qgep_model.wastewater_node) + if self.filtered: + query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + abwasserknoten = self.abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **self.qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **self.qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), + # --- abwasserknoten --- + hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + self.abwasser_session.add(abwasserknoten) + self.qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + def export_reach_point(self): query = self.qgep_session.query(self.qgep_model.reach_point) if self.filtered: From 953725151acb0f0cb6f6ec0e24e71877d564a7d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:14:57 +0000 Subject: [PATCH 189/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 323bb82e..22be8419 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -530,6 +530,7 @@ def export_wastewater_node(self): logger.info("done") self.abwasser_session.flush() + def export_wastewater_node_check_fk_in_subset(self): query = self.qgep_session.query(self.qgep_model.wastewater_node) if self.filtered: From 7035a406bdfb9a4087d8a43df4dfb5f57cb2c406 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 11:18:46 +0100 Subject: [PATCH 190/245] dssexport distinguish if filtered or not - check_fk_in_subset or not used --- qgepqwat2ili/qgepdss/export.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 5fde976a..d51c5f49 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1884,13 +1884,25 @@ def overflow_common(row): logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") qgep_export_utils.export_reach_point() - logger.info( - "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" - ) - qgep_export_utils.export_wastewater_node() + # with or without check_fk_in_subset + if filtered: + logger.info( + "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + ) + qgep_export_utils.export_wastewater_node_check_fk_in_subset() + + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + qgep_export_utils.export_reach_check_fk_in_subset() + + else: + logger.info( + "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + ) + qgep_export_utils.export_wastewater_node() + + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + qgep_export_utils.export_reach() - logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") - qgep_export_utils.export_reach() logger.info( "Exporting QGEP.profile_geometry -> ABWASSER.rohrprofil_geometrie, ABWASSER.metaattribute" From 2736e6727f4fdb737d0b810129debb83e5b7b427 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:19:14 +0000 Subject: [PATCH 191/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index d51c5f49..19dd27d0 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1903,7 +1903,6 @@ def overflow_common(row): logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() - logger.info( "Exporting QGEP.profile_geometry -> ABWASSER.rohrprofil_geometrie, ABWASSER.metaattribute" ) From dd52931c565ffcaa6168a7b0834bc6b16f3aa01d Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 12:33:54 +0100 Subject: [PATCH 192/245] correct intend --- qgepqwat2ili/utils/qgep_export_utils.py | 156 ++++++++++++------------ 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 22be8419..89beae9a 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -490,86 +490,86 @@ def export_pipe_profile(self): self.abwasser_session.flush() -def export_wastewater_node(self): - query = self.qgep_session.query(self.qgep_model.wastewater_node) - if self.filtered: - query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # --- wastewater_node --- - - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - abwasserknoten = self.abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **self.qgep_export_utils.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **self.qgep_export_utils.wastewater_networkelement_common(row), - # --- abwasserknoten --- - hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, - ) - self.abwasser_session.add(abwasserknoten) - self.qgep_export_utils.create_metaattributes(row) - print(".", end="") - logger.info("done") - self.abwasser_session.flush() + def export_wastewater_node(self): + query = self.qgep_session.query(self.qgep_model.wastewater_node) + if self.filtered: + query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark -def export_wastewater_node_check_fk_in_subset(self): - query = self.qgep_session.query(self.qgep_model.wastewater_node) - if self.filtered: - query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # --- wastewater_node --- - - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - abwasserknoten = self.abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **self.qgep_export_utils.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **self.qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), - # --- abwasserknoten --- - hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, - ) - self.abwasser_session.add(abwasserknoten) - self.qgep_export_utils.create_metaattributes(row) - print(".", end="") - logger.info("done") - self.abwasser_session.flush() + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + abwasserknoten = self.abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **self.qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **self.qgep_export_utils.wastewater_networkelement_common(row), + # --- abwasserknoten --- + hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + self.abwasser_session.add(abwasserknoten) + self.qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + + + def export_wastewater_node_check_fk_in_subset(self): + query = self.qgep_session.query(self.qgep_model.wastewater_node) + if self.filtered: + query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + abwasserknoten = self.abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **self.qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **self.qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), + # --- abwasserknoten --- + hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + self.abwasser_session.add(abwasserknoten) + self.qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() def export_reach_point(self): query = self.qgep_session.query(self.qgep_model.reach_point) From e998d8f5d1df08f22eaf2c8c7b5f6e9a8de2a2b6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:34:32 +0000 Subject: [PATCH 193/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 89beae9a..21dcce46 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -489,11 +489,12 @@ def export_pipe_profile(self): logger.info("done") self.abwasser_session.flush() - def export_wastewater_node(self): query = self.qgep_session.query(self.qgep_model.wastewater_node) if self.filtered: - query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = query.filter( + self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + ) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -530,11 +531,12 @@ def export_wastewater_node(self): logger.info("done") self.abwasser_session.flush() - def export_wastewater_node_check_fk_in_subset(self): query = self.qgep_session.query(self.qgep_model.wastewater_node) if self.filtered: - query = query.filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + query = query.filter( + self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + ) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From cdce83fe7f00b387cfa53abcbe7c51c2ebcf1b6f Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 12:43:38 +0100 Subject: [PATCH 194/245] remove qgep_export_utils --- qgepqwat2ili/utils/qgep_export_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 89beae9a..24e179c5 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -515,17 +515,17 @@ def export_wastewater_node(self): # FIELDS TO MAP TO ABWASSER.abwasserknoten # --- baseclass --- # --- sia405_baseclass --- - **self.qgep_export_utils.base_common(row, "abwasserknoten"), + **self.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- - **self.qgep_export_utils.wastewater_networkelement_common(row), + **self.wastewater_networkelement_common(row), # --- abwasserknoten --- - hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), lage=ST_Force2D(row.situation_geometry), rueckstaukote=row.backflow_level, sohlenkote=row.bottom_level, ) self.abwasser_session.add(abwasserknoten) - self.qgep_export_utils.create_metaattributes(row) + self.create_metaattributes(row) print(".", end="") logger.info("done") self.abwasser_session.flush() From a0d57678ba3aab3cc279437d18b964602f248c17 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 12:52:03 +0100 Subject: [PATCH 195/245] create def export_reach_point_check_fk_in_subset --- qgepqwat2ili/utils/qgep_export_utils.py | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index dd697c94..d7d4afc2 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -574,6 +574,51 @@ def export_wastewater_node_check_fk_in_subset(self): self.abwasser_session.flush() def export_reach_point(self): + query = self.qgep_session.query(self.qgep_model.reach_point) + if self.filtered: + query = query.join( + self.qgep_model.reach, + or_( + self.qgep_model.reach_point.obj_id + == self.qgep_model.reach.fk_reach_point_from, + self.qgep_model.reach_point.obj_id == self.qgep_model.reach.fk_reach_point_to, + ), + ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + for row in query: + + # AVAILABLE FIELDS IN QGEP.reach_point + + # --- reach_point --- + # elevation_accuracy, fk_dataowner, fk_provider, fk_wastewater_networkelement, identifier, last_modification, level, obj_id, outlet_shape, position_of_connection, remark, situation_geometry + + # --- _bwrel_ --- + # examination__BWREL_fk_reach_point, reach__BWREL_fk_reach_point_from, reach__BWREL_fk_reach_point_to + + # --- _rel_ --- + # elevation_accuracy__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_networkelement__REL, outlet_shape__REL + + haltungspunkt = self.abwasser_model.haltungspunkt( + # FIELDS TO MAP TO ABWASSER.haltungspunkt + # --- baseclass --- + # --- sia405_baseclass --- + **self.base_common(row, "haltungspunkt"), + # --- haltungspunkt --- + abwassernetzelementref=self.get_tid(row.fk_wastewater_networkelement__REL), + auslaufform=self.get_vl(row.outlet_shape__REL), + bemerkung=self.truncate(self.emptystr_to_null(row.remark), 80), + bezeichnung=self.null_to_emptystr(row.identifier), + hoehengenauigkeit=self.get_vl(row.elevation_accuracy__REL), + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lage_anschluss=row.position_of_connection, + ) + self.abwasser_session.add(haltungspunkt) + self.create_metaattributes(row) + print(".", end="") + logger.info("done") + self.abwasser_session.flush() + + def export_reach_point_check_fk_in_subset(self): query = self.qgep_session.query(self.qgep_model.reach_point) if self.filtered: query = query.join( From f19f2afd8c0dc7da8cd3be22ce136435c28445ab Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 12:54:19 +0100 Subject: [PATCH 196/245] adapt dss export include export_reach_point with / without check_fk_in_subset --- qgepqwat2ili/qgepdss/export.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 19dd27d0..2197ef8f 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1881,11 +1881,13 @@ def overflow_common(row): logger.info("done") abwasser_session.flush() - logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") - qgep_export_utils.export_reach_point() + # with or without check_fk_in_subset if filtered: + logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + qgep_export_utils.export_reach_point_check_fk_in_subset() + logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) @@ -1895,6 +1897,9 @@ def overflow_common(row): qgep_export_utils.export_reach_check_fk_in_subset() else: + logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + qgep_export_utils.export_reach_point() + logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) From e5e639c3f68ec73020705507b46ccc0c1199f4a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:54:35 +0000 Subject: [PATCH 197/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 2197ef8f..5e919208 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1881,13 +1881,11 @@ def overflow_common(row): logger.info("done") abwasser_session.flush() - - # with or without check_fk_in_subset if filtered: logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") qgep_export_utils.export_reach_point_check_fk_in_subset() - + logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) @@ -1899,7 +1897,7 @@ def overflow_common(row): else: logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") qgep_export_utils.export_reach_point() - + logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) From d6749366bf4db07d7c944de2c4c4eb689d1ad507 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 13:03:44 +0100 Subject: [PATCH 198/245] delete qgep_export_utils --- qgepqwat2ili/utils/qgep_export_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d7d4afc2..6a4da231 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -558,17 +558,17 @@ def export_wastewater_node_check_fk_in_subset(self): # FIELDS TO MAP TO ABWASSER.abwasserknoten # --- baseclass --- # --- sia405_baseclass --- - **self.qgep_export_utils.base_common(row, "abwasserknoten"), + **self.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- - **self.qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), + **self.wastewater_networkelement_common_check_fk_in_subset(row), # --- abwasserknoten --- - hydr_geometrieref=self.qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), lage=ST_Force2D(row.situation_geometry), rueckstaukote=row.backflow_level, sohlenkote=row.bottom_level, ) self.abwasser_session.add(abwasserknoten) - self.qgep_export_utils.create_metaattributes(row) + self.create_metaattributes(row) print(".", end="") logger.info("done") self.abwasser_session.flush() From 23ee6ce79a63b70e4fef7108452646c0a83f557f Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 13:08:10 +0100 Subject: [PATCH 199/245] adapt export sia405 --- qgepqwat2ili/qgepsia405/export.py | 59 +++++++++---------------------- 1 file changed, 17 insertions(+), 42 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 22f67c5c..43fa276f 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -312,55 +312,30 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("Exporting QGEP.pipe_profile -> ABWASSER.rohrprofil, ABWASSER.metaattribute") qgep_export_utils.export_pipe_profile() - logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") - qgep_export_utils.export_reach_point() - - logger.info( - "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" - ) - query = qgep_session.query(qgep_model.wastewater_node) + # with or without check_fk_in_subset if filtered: - query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + qgep_export_utils.export_reach_point_check_fk_in_subset() - # --- wastewater_node --- - - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + logger.info( + "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + ) + qgep_export_utils.export_wastewater_node_check_fk_in_subset() - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + qgep_export_utils.export_reach_check_fk_in_subset() - # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + else: + logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + qgep_export_utils.export_reach_point() - abwasserknoten = abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **qgep_export_utils.wastewater_networkelement_common(row), - # --- abwasserknoten --- - # TODO : WARNING : fk_hydr_geometry is not mapped - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, + logger.info( + "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) - abwasser_session.add(abwasserknoten) - qgep_export_utils.create_metaattributes(row) - print(".", end="") - logger.info("done") - abwasser_session.flush() + qgep_export_utils.export_wastewater_node() - logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") - qgep_export_utils.export_reach() + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + qgep_export_utils.export_reach() if ws_off_sia405abwasser: logger.info( From daa9a485f7905322a6bcad3170775727b18530f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:08:28 +0000 Subject: [PATCH 200/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 43fa276f..fab6cd08 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -1,6 +1,5 @@ import json -from geoalchemy2.functions import ST_Force2D from sqlalchemy.orm import Session from sqlalchemy.sql import text From 8395262aaf7f389a759751ffc839d298af85f316 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 13:14:12 +0100 Subject: [PATCH 201/245] add query.statement --- qgepqwat2ili/utils/qgep_export_utils.py | 44 +++++++++++++++++++++---- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 6a4da231..7e9dd19e 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -495,9 +495,9 @@ def export_wastewater_node(self): query = query.filter( self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) ) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.wastewater_node @@ -537,9 +537,9 @@ def export_wastewater_node_check_fk_in_subset(self): query = query.filter( self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) ) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.wastewater_node @@ -584,6 +584,9 @@ def export_reach_point(self): self.qgep_model.reach_point.obj_id == self.qgep_model.reach.fk_reach_point_to, ), ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.reach_point @@ -629,6 +632,9 @@ def export_reach_point_check_fk_in_subset(self): self.qgep_model.reach_point.obj_id == self.qgep_model.reach.fk_reach_point_to, ), ).filter(self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.reach_point @@ -673,6 +679,9 @@ def export_reach(self): query = query.filter( self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.reach @@ -731,6 +740,9 @@ def export_reach_check_fk_in_subset(self): query = query.filter( self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.reach @@ -796,6 +808,10 @@ def export_dryweather_downspout(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout @@ -833,7 +849,9 @@ def export_dryweather_downspout_ws_off_sia405abwasser(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) - + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_downspout @@ -878,6 +896,9 @@ def export_access_aid(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.access_aid @@ -915,6 +936,9 @@ def export_access_aid_ws_off_sia405abwasser(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.access_aid @@ -959,6 +983,9 @@ def export_dryweather_flume(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_flume @@ -996,6 +1023,9 @@ def export_dryweather_flume_ws_off_sia405abwasser(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.dryweather_flume From 51dcb08889560dd56eb99491de5071690c3df18a Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 13:24:57 +0100 Subject: [PATCH 202/245] more --- qgepqwat2ili/utils/qgep_export_utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 7e9dd19e..1bb9f4b9 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1020,8 +1020,10 @@ def export_dryweather_flume(self): def export_dryweather_flume_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) # if ws_off_sia405abwasser always filter out with subset_wws_ids - query = query.join(self.qgep_model.wastewater_structure).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) + query = ( + query.join(self.qgep_model.wastewater_structure) + .filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) ) # add sql statement to logger statement = query.statement @@ -1070,6 +1072,9 @@ def export_cover(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.cover @@ -1116,6 +1121,9 @@ def export_cover_ws_off_sia405abwasser(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.cover @@ -1169,6 +1177,9 @@ def export_benching(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.benching @@ -1206,6 +1217,9 @@ def export_benching_ws_off_sia405abwasser(self): query = query.join(self.qgep_model.wastewater_structure).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.benching From 56422b607f83a84098e63c057dc709b7343b8e6d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:15:33 +0000 Subject: [PATCH 203/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 1bb9f4b9..80ae9fe7 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1020,10 +1020,8 @@ def export_dryweather_flume(self): def export_dryweather_flume_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.dryweather_flume) # if ws_off_sia405abwasser always filter out with subset_wws_ids - query = ( - query.join(self.qgep_model.wastewater_structure) - .filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) + query = query.join(self.qgep_model.wastewater_structure).filter( + self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) # add sql statement to logger statement = query.statement From f5c433f8daed1f9805c568a74afb71730bc83347 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 14:35:11 +0100 Subject: [PATCH 204/245] add if filtered for messstelle --- qgepqwat2ili/qgepdss/export.py | 74 +++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 5e919208..ab3a75b7 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2795,29 +2795,57 @@ def overflow_common(row): # --- _rel_ --- # to do add relations fk_dataowner__REL, fk_provider__REL, profile_type__REL - messstelle = abwasser_model.messstelle( - # FIELDS TO MAP TO ABWASSER.messstelle - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "messstelle"), - # --- messstelle --- - abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), - abwasserreinigungsanlageref=qgep_export_utils.get_tid( - row.fk_waste_water_treatment_plant__REL - ), - art=row.kind, - bemerkung=qgep_export_utils.truncate( - qgep_export_utils.emptystr_to_null(row.remark), 80 - ), - betreiberref=qgep_export_utils.get_tid(row.fk_operator__REL), - bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), - gewaesserabschnittref=qgep_export_utils.get_tid(row.fk_water_course_segment__REL), - lage=ST_Force2D(row.situation_geometry), - # not supported in qgep datamodel yet, reference on same class - # referenzstelleref=qgep_export_utils.get_tid(row.fk_reference_station__REL), - staukoerper=qgep_export_utils.get_vl(row.damming_device__REL), - zweck=qgep_export_utils.get_vl(row.purpose__REL), - ) + if filtered: + messstelle = abwasser_model.messstelle( + # FIELDS TO MAP TO ABWASSER.messstelle + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "messstelle"), + # --- messstelle --- + #abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid ( + subset_wws_ids, row.fk_wastewater_structure__REL + ), + abwasserreinigungsanlageref=qgep_export_utils.get_tid( + row.fk_waste_water_treatment_plant__REL + ), + art=row.kind, + bemerkung=qgep_export_utils.truncate( + qgep_export_utils.emptystr_to_null(row.remark), 80 + ), + betreiberref=qgep_export_utils.get_tid(row.fk_operator__REL), + bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), + gewaesserabschnittref=qgep_export_utils.get_tid(row.fk_water_course_segment__REL), + lage=ST_Force2D(row.situation_geometry), + # not supported in qgep datamodel yet, reference on same class + # referenzstelleref=qgep_export_utils.get_tid(row.fk_reference_station__REL), + staukoerper=qgep_export_utils.get_vl(row.damming_device__REL), + zweck=qgep_export_utils.get_vl(row.purpose__REL), + ) + else: + messstelle = abwasser_model.messstelle( + # FIELDS TO MAP TO ABWASSER.messstelle + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "messstelle"), + # --- messstelle --- + abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), + abwasserreinigungsanlageref=qgep_export_utils.get_tid( + row.fk_waste_water_treatment_plant__REL + ), + art=row.kind, + bemerkung=qgep_export_utils.truncate( + qgep_export_utils.emptystr_to_null(row.remark), 80 + ), + betreiberref=qgep_export_utils.get_tid(row.fk_operator__REL), + bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), + gewaesserabschnittref=qgep_export_utils.get_tid(row.fk_water_course_segment__REL), + lage=ST_Force2D(row.situation_geometry), + # not supported in qgep datamodel yet, reference on same class + # referenzstelleref=qgep_export_utils.get_tid(row.fk_reference_station__REL), + staukoerper=qgep_export_utils.get_vl(row.damming_device__REL), + zweck=qgep_export_utils.get_vl(row.purpose__REL), + ) abwasser_session.add(messstelle) qgep_export_utils.create_metaattributes(row) print(".", end="") From 5dde20541192c95f4212e362fda465deec0eb214 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 14:37:58 +0100 Subject: [PATCH 205/245] correct call - add qgep_export_utils --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index ab3a75b7..dd32783e 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2803,8 +2803,8 @@ def overflow_common(row): **qgep_export_utils.base_common(row, "messstelle"), # --- messstelle --- #abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), - "abwasserbauwerkref": check_fk_in_subsetid ( - subset_wws_ids, row.fk_wastewater_structure__REL + "abwasserbauwerkref": qgep_export_utils.check_fk_in_subsetid ( + subset_wws_ids, row.fk_wastewater_structure__REL ), abwasserreinigungsanlageref=qgep_export_utils.get_tid( row.fk_waste_water_treatment_plant__REL From 8861ece4a64f246f534a22557ff90d0b2ebcce7b Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 14:39:39 +0100 Subject: [PATCH 206/245] correction --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index dd32783e..25f2c848 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2803,7 +2803,7 @@ def overflow_common(row): **qgep_export_utils.base_common(row, "messstelle"), # --- messstelle --- #abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), - "abwasserbauwerkref": qgep_export_utils.check_fk_in_subsetid ( + abwasserbauwerkref=qgep_export_utils.check_fk_in_subsetid( subset_wws_ids, row.fk_wastewater_structure__REL ), abwasserreinigungsanlageref=qgep_export_utils.get_tid( From 834b387ea5bcc4e3ba2ac29bf12a2cfa2bc50c99 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:39:55 +0000 Subject: [PATCH 207/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 25f2c848..626bb696 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2796,13 +2796,13 @@ def overflow_common(row): # to do add relations fk_dataowner__REL, fk_provider__REL, profile_type__REL if filtered: - messstelle = abwasser_model.messstelle( + messstelle = abwasser_model.messstelle( # FIELDS TO MAP TO ABWASSER.messstelle # --- baseclass --- # --- sia405_baseclass --- **qgep_export_utils.base_common(row, "messstelle"), # --- messstelle --- - #abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), + # abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), abwasserbauwerkref=qgep_export_utils.check_fk_in_subsetid( subset_wws_ids, row.fk_wastewater_structure__REL ), From 7c83a41b57773de830b4aad05e5d78da0e3f5fbe Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 14:51:49 +0100 Subject: [PATCH 208/245] Specify fk_relation between cover and wws --- qgepqwat2ili/utils/qgep_export_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index 80ae9fe7..d6b8ecda 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1067,7 +1067,7 @@ def export_cover(self): # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = query.join(self.qgep_model.wastewater_structure).filter( + query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure = wastewater_structure.obj_id).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) # add sql statement to logger @@ -1116,7 +1116,7 @@ def export_cover(self): def export_cover_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.cover) # if ws_off_sia405abwasser always filter out with subset_wws_ids - query = query.join(self.qgep_model.wastewater_structure).filter( + query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure = wastewater_structure.obj_id).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) # add sql statement to logger From f66f76f213a943f36e4797eefe535af55b4e06de Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 14:55:18 +0100 Subject: [PATCH 209/245] correct == --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d6b8ecda..e6e9276f 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1067,7 +1067,7 @@ def export_cover(self): # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure = wastewater_structure.obj_id).filter( + query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure == wastewater_structure.obj_id).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) # add sql statement to logger From f4f0f9bb01241ec1d4ec3a5038f699dc7ebda05e Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 14:58:22 +0100 Subject: [PATCH 210/245] same --- qgepqwat2ili/utils/qgep_export_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index e6e9276f..e05eb855 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1116,7 +1116,7 @@ def export_cover(self): def export_cover_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.cover) # if ws_off_sia405abwasser always filter out with subset_wws_ids - query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure = wastewater_structure.obj_id).filter( + query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure == wastewater_structure.obj_id).filter( self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) ) # add sql statement to logger From 0ebb817566f963b5f5374f13dc5e01921553e06b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:58:45 +0000 Subject: [PATCH 211/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index e05eb855..ff242cba 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1067,9 +1067,10 @@ def export_cover(self): # ) # filtering only on wastewater_structures that are in subset_wws_ids if self.filtered: - query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure == wastewater_structure.obj_id).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + query = query.join( + self.qgep_model.wastewater_structure, + cover.fk_wastewater_structure == wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") @@ -1116,9 +1117,10 @@ def export_cover(self): def export_cover_ws_off_sia405abwasser(self): query = self.qgep_session.query(self.qgep_model.cover) # if ws_off_sia405abwasser always filter out with subset_wws_ids - query = query.join(self.qgep_model.wastewater_structure, cover.fk_wastewater_structure == wastewater_structure.obj_id).filter( - self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids) - ) + query = query.join( + self.qgep_model.wastewater_structure, + cover.fk_wastewater_structure == wastewater_structure.obj_id, + ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement logger.debug(f" selection query = {statement}") From f317ed59e5360acbc3bbade2ca929374f3e3cefb Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 17:32:26 +0100 Subject: [PATCH 212/245] add self.qgep_model. --- qgepqwat2ili/utils/qgep_export_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index ff242cba..ea919b14 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1069,7 +1069,7 @@ def export_cover(self): if self.filtered: query = query.join( self.qgep_model.wastewater_structure, - cover.fk_wastewater_structure == wastewater_structure.obj_id, + self.qgep_model.cover.fk_wastewater_structure == self.qgep_model.wastewater_structure.obj_id, ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement @@ -1119,7 +1119,7 @@ def export_cover_ws_off_sia405abwasser(self): # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join( self.qgep_model.wastewater_structure, - cover.fk_wastewater_structure == wastewater_structure.obj_id, + self.qgep_model.cover.fk_wastewater_structure == self.qgep_model.wastewater_structure.obj_id, ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement From 04e0a472373cb9eb7c321edccd212dd79fee4641 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:32:43 +0000 Subject: [PATCH 213/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/qgep_export_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index ea919b14..19404b22 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1069,7 +1069,8 @@ def export_cover(self): if self.filtered: query = query.join( self.qgep_model.wastewater_structure, - self.qgep_model.cover.fk_wastewater_structure == self.qgep_model.wastewater_structure.obj_id, + self.qgep_model.cover.fk_wastewater_structure + == self.qgep_model.wastewater_structure.obj_id, ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement @@ -1119,7 +1120,8 @@ def export_cover_ws_off_sia405abwasser(self): # if ws_off_sia405abwasser always filter out with subset_wws_ids query = query.join( self.qgep_model.wastewater_structure, - self.qgep_model.cover.fk_wastewater_structure == self.qgep_model.wastewater_structure.obj_id, + self.qgep_model.cover.fk_wastewater_structure + == self.qgep_model.wastewater_structure.obj_id, ).filter(self.qgep_model.wastewater_structure.obj_id.in_(self.subset_wws_ids)) # add sql statement to logger statement = query.statement From 656894dc2906bad8eb26af397808edf6e1dc1d58 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 18:00:54 +0100 Subject: [PATCH 214/245] move back export_wastewater_node_* to export, as difference in dss and sia405/kek --- qgepqwat2ili/qgepdss/export.py | 90 ++++++++++++- qgepqwat2ili/qgepsia405/export.py | 91 ++++++++++++- qgepqwat2ili/utils/qgep_export_utils.py | 166 ++++++++++++------------ 3 files changed, 260 insertions(+), 87 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 626bb696..30c68f97 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1889,8 +1889,52 @@ def overflow_common(row): logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) - qgep_export_utils.export_wastewater_node_check_fk_in_subset() + # cannot be moved to qgep_export_utils because fk_hydr_geometry is only in VSA-DSS but not in SIA405 Abwasser and KEK + # qgep_export_utils.export_wastewater_node_check_fk_in_subset() + + query = qgep_session.query(qgep_model.wastewater_node) + if filtered: + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + #**qgep_export_utils.wastewater_networkelement_common(row), + **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset + # --- abwasserknoten --- + hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach_check_fk_in_subset() @@ -1901,7 +1945,49 @@ def overflow_common(row): logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) - qgep_export_utils.export_wastewater_node() + # cannot be moved to qgep_export_utils because fk_hydr_geometry is only in VSA-DSS but not in SIA405 Abwasser and KEK + # qgep_export_utils.export_wastewater_node_check_fk_in_subset() + + query = qgep_session.query(qgep_model.wastewater_node) + if filtered: + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **qgep_export_utils.wastewater_networkelement_common(row), + # --- abwasserknoten --- + hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index fab6cd08..52dea8b0 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -319,7 +319,52 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) - qgep_export_utils.export_wastewater_node_check_fk_in_subset() + # cannot be moved to qgep_export_utils because fk_hydr_geometry is only in VSA-DSS but not in SIA405 Abwasser and KEK + # qgep_export_utils.export_wastewater_node_check_fk_in_subset() + + query = qgep_session.query(qgep_model.wastewater_node) + if filtered: + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + #**qgep_export_utils.wastewater_networkelement_common(row), + **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset + # --- abwasserknoten --- + # TODO : WARNING : fk_hydr_geometry is not mapped + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach_check_fk_in_subset() @@ -331,7 +376,49 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) - qgep_export_utils.export_wastewater_node() + # qgep_export_utils.export_wastewater_node() + + query = qgep_session.query(qgep_model.wastewater_node) + if filtered: + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **qgep_export_utils.wastewater_networkelement_common(row), + # --- abwasserknoten --- + # TODO : WARNING : fk_hydr_geometry is not mapped + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index ea919b14..943f7c68 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -489,89 +489,89 @@ def export_pipe_profile(self): logger.info("done") self.abwasser_session.flush() - def export_wastewater_node(self): - query = self.qgep_session.query(self.qgep_model.wastewater_node) - if self.filtered: - query = query.filter( - self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) - ) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # --- wastewater_node --- - - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - abwasserknoten = self.abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **self.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **self.wastewater_networkelement_common(row), - # --- abwasserknoten --- - hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, - ) - self.abwasser_session.add(abwasserknoten) - self.create_metaattributes(row) - print(".", end="") - logger.info("done") - self.abwasser_session.flush() - - def export_wastewater_node_check_fk_in_subset(self): - query = self.qgep_session.query(self.qgep_model.wastewater_node) - if self.filtered: - query = query.filter( - self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) - ) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # --- wastewater_node --- - - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - abwasserknoten = self.abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **self.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **self.wastewater_networkelement_common_check_fk_in_subset(row), - # --- abwasserknoten --- - hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, - ) - self.abwasser_session.add(abwasserknoten) - self.create_metaattributes(row) - print(".", end="") - logger.info("done") - self.abwasser_session.flush() + # def export_wastewater_node(self): + # query = self.qgep_session.query(self.qgep_model.wastewater_node) + # if self.filtered: + # query = query.filter( + # self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + # ) + # # add sql statement to logger + # statement = query.statement + # logger.debug(f" selection query = {statement}") + # for row in query: + # # AVAILABLE FIELDS IN QGEP.wastewater_node + + # # --- wastewater_networkelement --- + # # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # # --- wastewater_node --- + + # # --- _bwrel_ --- + # # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # abwasserknoten = self.abwasser_model.abwasserknoten( + # # FIELDS TO MAP TO ABWASSER.abwasserknoten + # # --- baseclass --- + # # --- sia405_baseclass --- + # **self.base_common(row, "abwasserknoten"), + # # --- abwassernetzelement --- + # **self.wastewater_networkelement_common(row), + # # --- abwasserknoten --- + # hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), + # lage=ST_Force2D(row.situation_geometry), + # rueckstaukote=row.backflow_level, + # sohlenkote=row.bottom_level, + # ) + # self.abwasser_session.add(abwasserknoten) + # self.create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # self.abwasser_session.flush() + + # def export_wastewater_node_check_fk_in_subset(self): + # query = self.qgep_session.query(self.qgep_model.wastewater_node) + # if self.filtered: + # query = query.filter( + # self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + # ) + # # add sql statement to logger + # statement = query.statement + # logger.debug(f" selection query = {statement}") + # for row in query: + # # AVAILABLE FIELDS IN QGEP.wastewater_node + + # # --- wastewater_networkelement --- + # # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # # --- wastewater_node --- + + # # --- _bwrel_ --- + # # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # abwasserknoten = self.abwasser_model.abwasserknoten( + # # FIELDS TO MAP TO ABWASSER.abwasserknoten + # # --- baseclass --- + # # --- sia405_baseclass --- + # **self.base_common(row, "abwasserknoten"), + # # --- abwassernetzelement --- + # **self.wastewater_networkelement_common_check_fk_in_subset(row), + # # --- abwasserknoten --- + # hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), + # lage=ST_Force2D(row.situation_geometry), + # rueckstaukote=row.backflow_level, + # sohlenkote=row.bottom_level, + # ) + # self.abwasser_session.add(abwasserknoten) + # self.create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # self.abwasser_session.flush() def export_reach_point(self): query = self.qgep_session.query(self.qgep_model.reach_point) From c5ca3efdffdb8b2b0e75eff6cbb1fa84aa4ee498 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:01:21 +0000 Subject: [PATCH 215/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 2 +- qgepqwat2ili/utils/qgep_export_utils.py | 160 ++++++++++++------------ 2 files changed, 81 insertions(+), 81 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 30c68f97..10249abd 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1934,7 +1934,7 @@ def overflow_common(row): print(".", end="") logger.info("done") abwasser_session.flush() - + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach_check_fk_in_subset() diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index d19748bf..e29d29a8 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -490,88 +490,88 @@ def export_pipe_profile(self): self.abwasser_session.flush() # def export_wastewater_node(self): - # query = self.qgep_session.query(self.qgep_model.wastewater_node) - # if self.filtered: - # query = query.filter( - # self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) - # ) - # # add sql statement to logger - # statement = query.statement - # logger.debug(f" selection query = {statement}") - # for row in query: - # # AVAILABLE FIELDS IN QGEP.wastewater_node - - # # --- wastewater_networkelement --- - # # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # # --- wastewater_node --- - - # # --- _bwrel_ --- - # # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # # --- _rel_ --- - # # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - # abwasserknoten = self.abwasser_model.abwasserknoten( - # # FIELDS TO MAP TO ABWASSER.abwasserknoten - # # --- baseclass --- - # # --- sia405_baseclass --- - # **self.base_common(row, "abwasserknoten"), - # # --- abwassernetzelement --- - # **self.wastewater_networkelement_common(row), - # # --- abwasserknoten --- - # hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), - # lage=ST_Force2D(row.situation_geometry), - # rueckstaukote=row.backflow_level, - # sohlenkote=row.bottom_level, - # ) - # self.abwasser_session.add(abwasserknoten) - # self.create_metaattributes(row) - # print(".", end="") - # logger.info("done") - # self.abwasser_session.flush() + # query = self.qgep_session.query(self.qgep_model.wastewater_node) + # if self.filtered: + # query = query.filter( + # self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + # ) + # # add sql statement to logger + # statement = query.statement + # logger.debug(f" selection query = {statement}") + # for row in query: + # # AVAILABLE FIELDS IN QGEP.wastewater_node + + # # --- wastewater_networkelement --- + # # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # # --- wastewater_node --- + + # # --- _bwrel_ --- + # # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # abwasserknoten = self.abwasser_model.abwasserknoten( + # # FIELDS TO MAP TO ABWASSER.abwasserknoten + # # --- baseclass --- + # # --- sia405_baseclass --- + # **self.base_common(row, "abwasserknoten"), + # # --- abwassernetzelement --- + # **self.wastewater_networkelement_common(row), + # # --- abwasserknoten --- + # hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), + # lage=ST_Force2D(row.situation_geometry), + # rueckstaukote=row.backflow_level, + # sohlenkote=row.bottom_level, + # ) + # self.abwasser_session.add(abwasserknoten) + # self.create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # self.abwasser_session.flush() # def export_wastewater_node_check_fk_in_subset(self): - # query = self.qgep_session.query(self.qgep_model.wastewater_node) - # if self.filtered: - # query = query.filter( - # self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) - # ) - # # add sql statement to logger - # statement = query.statement - # logger.debug(f" selection query = {statement}") - # for row in query: - # # AVAILABLE FIELDS IN QGEP.wastewater_node - - # # --- wastewater_networkelement --- - # # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # # --- wastewater_node --- - - # # --- _bwrel_ --- - # # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node - - # # --- _rel_ --- - # # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL - - # abwasserknoten = self.abwasser_model.abwasserknoten( - # # FIELDS TO MAP TO ABWASSER.abwasserknoten - # # --- baseclass --- - # # --- sia405_baseclass --- - # **self.base_common(row, "abwasserknoten"), - # # --- abwassernetzelement --- - # **self.wastewater_networkelement_common_check_fk_in_subset(row), - # # --- abwasserknoten --- - # hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), - # lage=ST_Force2D(row.situation_geometry), - # rueckstaukote=row.backflow_level, - # sohlenkote=row.bottom_level, - # ) - # self.abwasser_session.add(abwasserknoten) - # self.create_metaattributes(row) - # print(".", end="") - # logger.info("done") - # self.abwasser_session.flush() + # query = self.qgep_session.query(self.qgep_model.wastewater_node) + # if self.filtered: + # query = query.filter( + # self.qgep_model.wastewater_networkelement.obj_id.in_(self.subset_ids) + # ) + # # add sql statement to logger + # statement = query.statement + # logger.debug(f" selection query = {statement}") + # for row in query: + # # AVAILABLE FIELDS IN QGEP.wastewater_node + + # # --- wastewater_networkelement --- + # # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # # --- wastewater_node --- + + # # --- _bwrel_ --- + # # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # abwasserknoten = self.abwasser_model.abwasserknoten( + # # FIELDS TO MAP TO ABWASSER.abwasserknoten + # # --- baseclass --- + # # --- sia405_baseclass --- + # **self.base_common(row, "abwasserknoten"), + # # --- abwassernetzelement --- + # **self.wastewater_networkelement_common_check_fk_in_subset(row), + # # --- abwasserknoten --- + # hydr_geometrieref=self.get_tid(row.fk_hydr_geometry__REL), + # lage=ST_Force2D(row.situation_geometry), + # rueckstaukote=row.backflow_level, + # sohlenkote=row.bottom_level, + # ) + # self.abwasser_session.add(abwasserknoten) + # self.create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # self.abwasser_session.flush() def export_reach_point(self): query = self.qgep_session.query(self.qgep_model.reach_point) From 29a765a83fd7863bb452ce0424b2c633a7e93066 Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 18:04:24 +0100 Subject: [PATCH 216/245] missing (row), --- qgepqwat2ili/qgepdss/export.py | 2 +- qgepqwat2ili/qgepsia405/export.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 30c68f97..b9dab594 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1922,7 +1922,7 @@ def overflow_common(row): **qgep_export_utils.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- #**qgep_export_utils.wastewater_networkelement_common(row), - **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset + **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), # --- abwasserknoten --- hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), lage=ST_Force2D(row.situation_geometry), diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 52dea8b0..a12f4c29 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -351,7 +351,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske **qgep_export_utils.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- #**qgep_export_utils.wastewater_networkelement_common(row), - **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset + **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), # --- abwasserknoten --- # TODO : WARNING : fk_hydr_geometry is not mapped lage=ST_Force2D(row.situation_geometry), From 4b43d8f43e38ab3bf511d0d99053a5aeba540577 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:12:01 +0000 Subject: [PATCH 217/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 2 +- qgepqwat2ili/qgepsia405/export.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index e8534650..c12bad6b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -1921,7 +1921,7 @@ def overflow_common(row): # --- sia405_baseclass --- **qgep_export_utils.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- - #**qgep_export_utils.wastewater_networkelement_common(row), + # **qgep_export_utils.wastewater_networkelement_common(row), **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), # --- abwasserknoten --- hydr_geometrieref=qgep_export_utils.get_tid(row.fk_hydr_geometry__REL), diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index a12f4c29..d73f0499 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -350,7 +350,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # --- sia405_baseclass --- **qgep_export_utils.base_common(row, "abwasserknoten"), # --- abwassernetzelement --- - #**qgep_export_utils.wastewater_networkelement_common(row), + # **qgep_export_utils.wastewater_networkelement_common(row), **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), # --- abwasserknoten --- # TODO : WARNING : fk_hydr_geometry is not mapped @@ -364,8 +364,6 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("done") abwasser_session.flush() - - logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach_check_fk_in_subset() @@ -419,7 +417,6 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("done") abwasser_session.flush() - logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach() From 069a6b34de63a99a746295a2f6137bcc7252b55f Mon Sep 17 00:00:00 2001 From: SJiB Date: Tue, 17 Dec 2024 18:38:19 +0100 Subject: [PATCH 218/245] read from geoalchemy2.functions import ST_Force2D --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index a12f4c29..5f9c09a2 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -1,5 +1,5 @@ import json - +from geoalchemy2.functions import ST_Force2D from sqlalchemy.orm import Session from sqlalchemy.sql import text From 86c2954c9a4ad81a89f148e01538fbae30621c53 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:40:06 +0000 Subject: [PATCH 219/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9afb2bfe..f2936eeb 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -1,4 +1,5 @@ import json + from geoalchemy2.functions import ST_Force2D from sqlalchemy.orm import Session from sqlalchemy.sql import text From c140f28a4256a1afb482cbd114847ffc137dd923 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 11:08:15 +0100 Subject: [PATCH 220/245] export sia450 - if filtered and ws_off_sia405abwasser -> do check_fk_in_subset --- qgepqwat2ili/qgepsia405/export.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9afb2bfe..46b6c15c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -312,12 +312,13 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske qgep_export_utils.export_pipe_profile() # with or without check_fk_in_subset - if filtered: - logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + # if filtered + if filtered and ws_off_sia405abwasser: + logger.info("Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") qgep_export_utils.export_reach_point_check_fk_in_subset() logger.info( - "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + "Exporting QGEP.wastewater_node (check_fk_in_subset) -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" ) # cannot be moved to qgep_export_utils because fk_hydr_geometry is only in VSA-DSS but not in SIA405 Abwasser and KEK # qgep_export_utils.export_wastewater_node_check_fk_in_subset() @@ -364,9 +365,10 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("done") abwasser_session.flush() - logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + logger.info("Exporting QGEP.reach (check_fk_in_subset) -> ABWASSER.haltung, ABWASSER.metaattribute") qgep_export_utils.export_reach_check_fk_in_subset() + # not filtered and not ws_off_sia405abwasser else: logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") qgep_export_utils.export_reach_point() From 236970fd00d21321f8412e2e6e6454c2b28406a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:10:05 +0000 Subject: [PATCH 221/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 3385dad0..679bc204 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -315,7 +315,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # with or without check_fk_in_subset # if filtered if filtered and ws_off_sia405abwasser: - logger.info("Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute" + ) qgep_export_utils.export_reach_point_check_fk_in_subset() logger.info( @@ -366,7 +368,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info("done") abwasser_session.flush() - logger.info("Exporting QGEP.reach (check_fk_in_subset) -> ABWASSER.haltung, ABWASSER.metaattribute") + logger.info( + "Exporting QGEP.reach (check_fk_in_subset) -> ABWASSER.haltung, ABWASSER.metaattribute" + ) qgep_export_utils.export_reach_check_fk_in_subset() # not filtered and not ws_off_sia405abwasser From d2d54cc366177784d7a337913636620fead1d2e6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 11:24:46 +0100 Subject: [PATCH 222/245] criteria or instead of and --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 679bc204..1417ca5f 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -314,7 +314,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # with or without check_fk_in_subset # if filtered - if filtered and ws_off_sia405abwasser: + if filtered or ws_off_sia405abwasser: logger.info( "Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute" ) From ad1d24e88fd885a5c10981a8ac77371610335fb5 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 11:36:00 +0100 Subject: [PATCH 223/245] adapt export dss catchment_area --- qgepqwat2ili/qgepdss/export.py | 180 ++++++++++++++++++++++----------- 1 file changed, 119 insertions(+), 61 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index c12bad6b..e93a531b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2731,67 +2731,125 @@ def overflow_common(row): # --- _rel_ --- # to do add relations fk_dataowner__REL, fk_provider__REL, profile_type__REL - einzugsgebiet = abwasser_model.einzugsgebiet( - # FIELDS TO MAP TO ABWASSER.einzugsgebiet - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "einzugsgebiet"), - # --- einzugsgebiet --- - abflussbegrenzung_geplant=row.runoff_limit_planned, - abflussbegrenzung_ist=row.runoff_limit_current, - abflussbeiwert_rw_geplant=row.discharge_coefficient_rw_planned, - abflussbeiwert_rw_ist=row.discharge_coefficient_rw_current, - abflussbeiwert_sw_geplant=row.discharge_coefficient_ww_planned, - abflussbeiwert_sw_ist=row.discharge_coefficient_ww_current, - # changed call from qgep_export_utils.get_tid to qgep_export_utils.check_fk_in_subsetid so it does not write foreignkeys on elements that do not exist - abwassernetzelement_rw_geplantref=qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement_rw_planned__REL - ), - abwassernetzelement_rw_istref=qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement_rw_current__REL - ), - abwassernetzelement_sw_geplantref=qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement_ww_planned__REL - ), - abwassernetzelement_sw_istref=qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement_ww_current__REL - ), - befestigungsgrad_rw_geplant=row.seal_factor_rw_planned, - befestigungsgrad_rw_ist=row.seal_factor_rw_current, - befestigungsgrad_sw_geplant=row.seal_factor_ww_planned, - befestigungsgrad_sw_ist=row.seal_factor_ww_current, - bemerkung=qgep_export_utils.truncate( - qgep_export_utils.emptystr_to_null(row.remark), 80 - ), - bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), - direkteinleitung_in_gewaesser_geplant=qgep_export_utils.get_vl( - row.direct_discharge_planned__REL - ), - direkteinleitung_in_gewaesser_ist=qgep_export_utils.get_vl( - row.direct_discharge_current__REL - ), - einwohnerdichte_geplant=row.population_density_planned, - einwohnerdichte_ist=row.population_density_current, - entwaesserungssystem_geplant=qgep_export_utils.get_vl( - row.drainage_system_planned__REL - ), - entwaesserungssystem_ist=qgep_export_utils.get_vl(row.drainage_system_current__REL), - flaeche=row.surface_area, - fremdwasseranfall_geplant=row.sewer_infiltration_water_production_planned, - fremdwasseranfall_ist=row.sewer_infiltration_water_production_current, - perimeter=ST_Force2D(row.perimeter_geometry), - retention_geplant=qgep_export_utils.get_vl(row.retention_planned__REL), - retention_ist=qgep_export_utils.get_vl(row.retention_current__REL), - # sbw_*ref will be added with release 2020 - # sbw_rw_geplantref=qgep_export_utils.get_tid(row.fk_special_building_rw_planned__REL), - # sbw_rw_istref=qgep_export_utils.get_tid(row.fk_special_building_rw_current__REL), - # sbw_sw_geplantref=qgep_export_utils.get_tid(row.fk_special_building_ww_planned__REL), - # sbw_sw_istref=qgep_export_utils.get_tid(row.fk_special_building_ww_current__REL), - schmutzabwasseranfall_geplant=row.waste_water_production_planned, - schmutzabwasseranfall_ist=row.waste_water_production_current, - versickerung_geplant=qgep_export_utils.get_vl(row.infiltration_planned__REL), - versickerung_ist=qgep_export_utils.get_vl(row.infiltration_current__REL), - ) + + if filtered or ws_off_sia405abwasser: + einzugsgebiet = abwasser_model.einzugsgebiet( + # FIELDS TO MAP TO ABWASSER.einzugsgebiet + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "einzugsgebiet"), + # --- einzugsgebiet --- + abflussbegrenzung_geplant=row.runoff_limit_planned, + abflussbegrenzung_ist=row.runoff_limit_current, + abflussbeiwert_rw_geplant=row.discharge_coefficient_rw_planned, + abflussbeiwert_rw_ist=row.discharge_coefficient_rw_current, + abflussbeiwert_sw_geplant=row.discharge_coefficient_ww_planned, + abflussbeiwert_sw_ist=row.discharge_coefficient_ww_current, + # changed call from qgep_export_utils.get_tid to qgep_export_utils.check_fk_in_subsetid so it does not write foreignkeys on elements that do not exist + abwassernetzelement_rw_geplantref=qgep_export_utils.check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement_rw_planned__REL + ), + abwassernetzelement_rw_istref=qgep_export_utils.check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement_rw_current__REL + ), + abwassernetzelement_sw_geplantref=qgep_export_utils.check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement_ww_planned__REL + ), + abwassernetzelement_sw_istref=qgep_export_utils.check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement_ww_current__REL + ), + befestigungsgrad_rw_geplant=row.seal_factor_rw_planned, + befestigungsgrad_rw_ist=row.seal_factor_rw_current, + befestigungsgrad_sw_geplant=row.seal_factor_ww_planned, + befestigungsgrad_sw_ist=row.seal_factor_ww_current, + bemerkung=qgep_export_utils.truncate( + qgep_export_utils.emptystr_to_null(row.remark), 80 + ), + bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), + direkteinleitung_in_gewaesser_geplant=qgep_export_utils.get_vl( + row.direct_discharge_planned__REL + ), + direkteinleitung_in_gewaesser_ist=qgep_export_utils.get_vl( + row.direct_discharge_current__REL + ), + einwohnerdichte_geplant=row.population_density_planned, + einwohnerdichte_ist=row.population_density_current, + entwaesserungssystem_geplant=qgep_export_utils.get_vl( + row.drainage_system_planned__REL + ), + entwaesserungssystem_ist=qgep_export_utils.get_vl(row.drainage_system_current__REL), + flaeche=row.surface_area, + fremdwasseranfall_geplant=row.sewer_infiltration_water_production_planned, + fremdwasseranfall_ist=row.sewer_infiltration_water_production_current, + perimeter=ST_Force2D(row.perimeter_geometry), + retention_geplant=qgep_export_utils.get_vl(row.retention_planned__REL), + retention_ist=qgep_export_utils.get_vl(row.retention_current__REL), + # sbw_*ref will be added with release 2020 + # sbw_rw_geplantref=qgep_export_utils.get_tid(row.fk_special_building_rw_planned__REL), + # sbw_rw_istref=qgep_export_utils.get_tid(row.fk_special_building_rw_current__REL), + # sbw_sw_geplantref=qgep_export_utils.get_tid(row.fk_special_building_ww_planned__REL), + # sbw_sw_istref=qgep_export_utils.get_tid(row.fk_special_building_ww_current__REL), + schmutzabwasseranfall_geplant=row.waste_water_production_planned, + schmutzabwasseranfall_ist=row.waste_water_production_current, + versickerung_geplant=qgep_export_utils.get_vl(row.infiltration_planned__REL), + versickerung_ist=qgep_export_utils.get_vl(row.infiltration_current__REL), + ) + else: + einzugsgebiet = abwasser_model.einzugsgebiet( + # FIELDS TO MAP TO ABWASSER.einzugsgebiet + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "einzugsgebiet"), + # --- einzugsgebiet --- + abflussbegrenzung_geplant=row.runoff_limit_planned, + abflussbegrenzung_ist=row.runoff_limit_current, + abflussbeiwert_rw_geplant=row.discharge_coefficient_rw_planned, + abflussbeiwert_rw_ist=row.discharge_coefficient_rw_current, + abflussbeiwert_sw_geplant=row.discharge_coefficient_ww_planned, + abflussbeiwert_sw_ist=row.discharge_coefficient_ww_current, + # changed call from qgep_export_utils.get_tid to qgep_export_utils.check_fk_in_subsetid so it does not write foreignkeys on elements that do not exist + abwassernetzelement_rw_geplantref= + qgep_export_utils.get_tid(row.fk_wastewater_networkelement_rw_planned__REL), + abwassernetzelement_rw_istref=qgep_export_utils.get_tid(row. fk_wastewater_networkelement_rw_current__REL), + abwassernetzelement_sw_geplantref=qgep_export_utils.get_tid(row.fk_wastewater_networkelement_ww_planned__REL), + abwassernetzelement_sw_istref=qgep_export_utils.get_tid(row.fk_wastewater_networkelement_ww_current__REL ), + befestigungsgrad_rw_geplant=row.seal_factor_rw_planned, + befestigungsgrad_rw_ist=row.seal_factor_rw_current, + befestigungsgrad_sw_geplant=row.seal_factor_ww_planned, + befestigungsgrad_sw_ist=row.seal_factor_ww_current, + bemerkung=qgep_export_utils.truncate( + qgep_export_utils.emptystr_to_null(row.remark), 80 + ), + bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), + direkteinleitung_in_gewaesser_geplant=qgep_export_utils.get_vl( + row.direct_discharge_planned__REL + ), + direkteinleitung_in_gewaesser_ist=qgep_export_utils.get_vl( + row.direct_discharge_current__REL + ), + einwohnerdichte_geplant=row.population_density_planned, + einwohnerdichte_ist=row.population_density_current, + entwaesserungssystem_geplant=qgep_export_utils.get_vl( + row.drainage_system_planned__REL + ), + entwaesserungssystem_ist=qgep_export_utils.get_vl(row.drainage_system_current__REL), + flaeche=row.surface_area, + fremdwasseranfall_geplant=row.sewer_infiltration_water_production_planned, + fremdwasseranfall_ist=row.sewer_infiltration_water_production_current, + perimeter=ST_Force2D(row.perimeter_geometry), + retention_geplant=qgep_export_utils.get_vl(row.retention_planned__REL), + retention_ist=qgep_export_utils.get_vl(row.retention_current__REL), + # sbw_*ref will be added with release 2020 + # sbw_rw_geplantref=qgep_export_utils.get_tid(row.fk_special_building_rw_planned__REL), + # sbw_rw_istref=qgep_export_utils.get_tid(row.fk_special_building_rw_current__REL), + # sbw_sw_geplantref=qgep_export_utils.get_tid(row.fk_special_building_ww_planned__REL), + # sbw_sw_istref=qgep_export_utils.get_tid(row.fk_special_building_ww_current__REL), + schmutzabwasseranfall_geplant=row.waste_water_production_planned, + schmutzabwasseranfall_ist=row.waste_water_production_current, + versickerung_geplant=qgep_export_utils.get_vl(row.infiltration_planned__REL), + versickerung_ist=qgep_export_utils.get_vl(row.infiltration_current__REL), + ) + abwasser_session.add(einzugsgebiet) qgep_export_utils.create_metaattributes(row) print(".", end="") From 0412c60deac640a6f5c547a0b9dafea6592e6067 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:37:04 +0000 Subject: [PATCH 224/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index e93a531b..b816f88b 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -2731,7 +2731,6 @@ def overflow_common(row): # --- _rel_ --- # to do add relations fk_dataowner__REL, fk_provider__REL, profile_type__REL - if filtered or ws_off_sia405abwasser: einzugsgebiet = abwasser_model.einzugsgebiet( # FIELDS TO MAP TO ABWASSER.einzugsgebiet @@ -2777,7 +2776,9 @@ def overflow_common(row): entwaesserungssystem_geplant=qgep_export_utils.get_vl( row.drainage_system_planned__REL ), - entwaesserungssystem_ist=qgep_export_utils.get_vl(row.drainage_system_current__REL), + entwaesserungssystem_ist=qgep_export_utils.get_vl( + row.drainage_system_current__REL + ), flaeche=row.surface_area, fremdwasseranfall_geplant=row.sewer_infiltration_water_production_planned, fremdwasseranfall_ist=row.sewer_infiltration_water_production_current, @@ -2808,11 +2809,18 @@ def overflow_common(row): abflussbeiwert_sw_geplant=row.discharge_coefficient_ww_planned, abflussbeiwert_sw_ist=row.discharge_coefficient_ww_current, # changed call from qgep_export_utils.get_tid to qgep_export_utils.check_fk_in_subsetid so it does not write foreignkeys on elements that do not exist - abwassernetzelement_rw_geplantref= - qgep_export_utils.get_tid(row.fk_wastewater_networkelement_rw_planned__REL), - abwassernetzelement_rw_istref=qgep_export_utils.get_tid(row. fk_wastewater_networkelement_rw_current__REL), - abwassernetzelement_sw_geplantref=qgep_export_utils.get_tid(row.fk_wastewater_networkelement_ww_planned__REL), - abwassernetzelement_sw_istref=qgep_export_utils.get_tid(row.fk_wastewater_networkelement_ww_current__REL ), + abwassernetzelement_rw_geplantref=qgep_export_utils.get_tid( + row.fk_wastewater_networkelement_rw_planned__REL + ), + abwassernetzelement_rw_istref=qgep_export_utils.get_tid( + row.fk_wastewater_networkelement_rw_current__REL + ), + abwassernetzelement_sw_geplantref=qgep_export_utils.get_tid( + row.fk_wastewater_networkelement_ww_planned__REL + ), + abwassernetzelement_sw_istref=qgep_export_utils.get_tid( + row.fk_wastewater_networkelement_ww_current__REL + ), befestigungsgrad_rw_geplant=row.seal_factor_rw_planned, befestigungsgrad_rw_ist=row.seal_factor_rw_current, befestigungsgrad_sw_geplant=row.seal_factor_ww_planned, @@ -2832,7 +2840,9 @@ def overflow_common(row): entwaesserungssystem_geplant=qgep_export_utils.get_vl( row.drainage_system_planned__REL ), - entwaesserungssystem_ist=qgep_export_utils.get_vl(row.drainage_system_current__REL), + entwaesserungssystem_ist=qgep_export_utils.get_vl( + row.drainage_system_current__REL + ), flaeche=row.surface_area, fremdwasseranfall_geplant=row.sewer_infiltration_water_production_planned, fremdwasseranfall_ist=row.sewer_infiltration_water_production_current, From 156f25c544162359a50b7623aff159753faa40fa Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 11:58:50 +0100 Subject: [PATCH 225/245] export dss common_object_check_fk_in_subset added --- qgepqwat2ili/qgepdss/export.py | 172 ++++++++++++++++++++++++--------- 1 file changed, 124 insertions(+), 48 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index e93a531b..5f35d040 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -275,6 +275,26 @@ def connection_object_common(row): "fremdwasseranfall": row.sewer_infiltration_water_production, } + def connection_object_common_check_fk_in_subset(row): + """ + Returns common attributes for connection_object + """ + return { + # "abwassernetzelementref": qgep_export_utils.get_tid( + # row.fk_wastewater_networkelement__REL + # ), + "abwassernetzelementref": qgep_export_utils.check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement_rw_planned__REL + ), + "bemerkung": qgep_export_utils.truncate( + qgep_export_utils.emptystr_to_null(row.remark), 80 + ), + "betreiberref": qgep_export_utils.get_tid(row.fk_operator__REL), + "bezeichnung": qgep_export_utils.null_to_emptystr(row.identifier), + "eigentuemerref": qgep_export_utils.get_tid(row.fk_owner__REL), + "fremdwasseranfall": row.sewer_infiltration_water_production, + } + def surface_runoff_parameters_common(row): """ Returns common attributes for surface_runoff_parameters @@ -2410,19 +2430,34 @@ def overflow_common(row): # --- _rel_ --- # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, - gebaeude = abwasser_model.gebaeude( - # FIELDS TO MAP TO ABWASSER.gebaeude - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "gebaeude"), - # --- anschlussobjekt --- - **connection_object_common(row), - # --- gebaeude --- - hausnummer=row.house_number, - perimeter=ST_Force2D(row.perimeter_geometry), - referenzpunkt=ST_Force2D(row.reference_point_geometry), - standortname=row.location_name, - ) + if filtered or ws_off_sia405abwasser: + gebaeude = abwasser_model.gebaeude( + # FIELDS TO MAP TO ABWASSER.gebaeude + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "gebaeude"), + # --- anschlussobjekt --- + **connection_object_common_check_fk_in_subset(row), + # --- gebaeude --- + hausnummer=row.house_number, + perimeter=ST_Force2D(row.perimeter_geometry), + referenzpunkt=ST_Force2D(row.reference_point_geometry), + standortname=row.location_name, + ) + else: + gebaeude = abwasser_model.gebaeude( + # FIELDS TO MAP TO ABWASSER.gebaeude + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "gebaeude"), + # --- anschlussobjekt --- + **connection_object_common(row), + # --- gebaeude --- + hausnummer=row.house_number, + perimeter=ST_Force2D(row.perimeter_geometry), + referenzpunkt=ST_Force2D(row.reference_point_geometry), + standortname=row.location_name, + ) abwasser_session.add(gebaeude) qgep_export_utils.create_metaattributes(row) print(".", end="") @@ -2453,17 +2488,30 @@ def overflow_common(row): # --- _rel_ --- # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, - reservoir = abwasser_model.reservoir( - # FIELDS TO MAP TO ABWASSER.reservoir - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "reservoir"), - # --- anschlussobjekt --- - **connection_object_common(row), - # --- reservoir --- - lage=ST_Force2D(row.situation_geometry), - standortname=row.location_name, - ) + if filtered or ws_off_sia405abwasser: + reservoir = abwasser_model.reservoir( + # FIELDS TO MAP TO ABWASSER.reservoir + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "reservoir"), + # --- anschlussobjekt --- + **connection_object_common_check_fk_in_subset(row), + # --- reservoir --- + lage=ST_Force2D(row.situation_geometry), + standortname=row.location_name, + ) + else: + reservoir = abwasser_model.reservoir( + # FIELDS TO MAP TO ABWASSER.reservoir + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "reservoir"), + # --- anschlussobjekt --- + **connection_object_common(row), + # --- reservoir --- + lage=ST_Force2D(row.situation_geometry), + standortname=row.location_name, + ) abwasser_session.add(reservoir) qgep_export_utils.create_metaattributes(row) print(".", end="") @@ -2496,19 +2544,34 @@ def overflow_common(row): # --- _rel_ --- # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, - einzelflaeche = abwasser_model.einzelflaeche( - # FIELDS TO MAP TO ABWASSER.einzelflaeche - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "einzelflaeche"), - # --- anschlussobjekt --- - **connection_object_common(row), - # --- einzelflaeche --- - befestigung=qgep_export_utils.get_vl(row.pavement__REL), - funktion=qgep_export_utils.get_vl(row.function__REL), - neigung=row.inclination, - perimeter=ST_Force2D(row.perimeter_geometry), - ) + if filtered or ws_off_sia405abwasser: + einzelflaeche = abwasser_model.einzelflaeche( + # FIELDS TO MAP TO ABWASSER.einzelflaeche + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "einzelflaeche"), + # --- anschlussobjekt --- + **connection_object_common_check_fk_in_subset(row), + # --- einzelflaeche --- + befestigung=qgep_export_utils.get_vl(row.pavement__REL), + funktion=qgep_export_utils.get_vl(row.function__REL), + neigung=row.inclination, + perimeter=ST_Force2D(row.perimeter_geometry), + ) + else: + einzelflaeche = abwasser_model.einzelflaeche( + # FIELDS TO MAP TO ABWASSER.einzelflaeche + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "einzelflaeche"), + # --- anschlussobjekt --- + **connection_object_common(row), + # --- einzelflaeche --- + befestigung=qgep_export_utils.get_vl(row.pavement__REL), + funktion=qgep_export_utils.get_vl(row.function__REL), + neigung=row.inclination, + perimeter=ST_Force2D(row.perimeter_geometry), + ) abwasser_session.add(einzelflaeche) qgep_export_utils.create_metaattributes(row) print(".", end="") @@ -2539,17 +2602,30 @@ def overflow_common(row): # --- _rel_ --- # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, - brunnen = abwasser_model.brunnen( - # FIELDS TO MAP TO ABWASSER.brunnen - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "brunnen"), - # --- anschlussobjekt --- - **connection_object_common(row), - # --- brunnen --- - lage=ST_Force2D(row.situation_geometry), - standortname=row.location_name, - ) + if filtered or ws_off_sia405abwasser: + brunnen = abwasser_model.brunnen( + # FIELDS TO MAP TO ABWASSER.brunnen + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "brunnen"), + # --- anschlussobjekt --- + **connection_object_common_check_fk_in_subset(row), + # --- brunnen --- + lage=ST_Force2D(row.situation_geometry), + standortname=row.location_name, + ) + else: + brunnen = abwasser_model.brunnen( + # FIELDS TO MAP TO ABWASSER.brunnen + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "brunnen"), + # --- anschlussobjekt --- + **connection_object_common(row), + # --- brunnen --- + lage=ST_Force2D(row.situation_geometry), + standortname=row.location_name, + ) abwasser_session.add(brunnen) qgep_export_utils.create_metaattributes(row) print(".", end="") From ba67f9e4f42513052b0e2eaeebf97b426e692e05 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:59:24 +0000 Subject: [PATCH 226/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 9706afea..04abc560 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -281,11 +281,11 @@ def connection_object_common_check_fk_in_subset(row): """ return { # "abwassernetzelementref": qgep_export_utils.get_tid( - # row.fk_wastewater_networkelement__REL + # row.fk_wastewater_networkelement__REL # ), "abwassernetzelementref": qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement_rw_planned__REL - ), + subset_ids, row.fk_wastewater_networkelement_rw_planned__REL + ), "bemerkung": qgep_export_utils.truncate( qgep_export_utils.emptystr_to_null(row.remark), 80 ), From d0eda8dc3b8fb2bda3fc68667ffd541cdfcde250 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 12:20:57 +0100 Subject: [PATCH 227/245] export dss wrong fk name --- qgepqwat2ili/qgepdss/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 9706afea..3fe89df6 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -284,7 +284,7 @@ def connection_object_common_check_fk_in_subset(row): # row.fk_wastewater_networkelement__REL # ), "abwassernetzelementref": qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement_rw_planned__REL + subset_ids, row.fk_wastewater_networkelement__REL ), "bemerkung": qgep_export_utils.truncate( qgep_export_utils.emptystr_to_null(row.remark), 80 From 2e30705778e3f36827a31529c47661a4f5f4dd36 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:22:28 +0000 Subject: [PATCH 228/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 3e81da2b..b56af6d7 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -284,8 +284,8 @@ def connection_object_common_check_fk_in_subset(row): # row.fk_wastewater_networkelement__REL # ), "abwassernetzelementref": qgep_export_utils.check_fk_in_subsetid( - subset_ids, row.fk_wastewater_networkelement__REL - ), + subset_ids, row.fk_wastewater_networkelement__REL + ), "bemerkung": qgep_export_utils.truncate( qgep_export_utils.emptystr_to_null(row.remark), 80 ), From a31c9497e72193ff1bbe2452875ab7293b46addd Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:01:02 +0100 Subject: [PATCH 229/245] add logger.debug --- qgepqwat2ili/qgepsia405/export.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 1417ca5f..1514f058 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -55,7 +55,14 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ws_off_sia405abwasser = False # 1. Filtering - check if selection filtered = selection is not None + + # Logging for debugging + logger.debug(f"print filtered '{str(filtered)}'") + subset_ids = selection if selection is not None else [] + # Logging for debugging + logger.debug(f"print subset_ids: '{str(subset_ids)}'") + subset_wws_ids = [] if filtered: From 0fb6093dbb9504ee059a4ca9aa2129de5522659c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:01:18 +0000 Subject: [PATCH 230/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 1514f058..20c9829f 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -58,7 +58,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # Logging for debugging logger.debug(f"print filtered '{str(filtered)}'") - + subset_ids = selection if selection is not None else [] # Logging for debugging logger.debug(f"print subset_ids: '{str(subset_ids)}'") From 4e31b696388a2f9786fdd852b55fd37343dbfd4e Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:01:40 +0100 Subject: [PATCH 231/245] turn off automatic extend to neighbouring wwn of reaches --- qgepqwat2ili/gui/__init__.py | 4 +++- qgepqwat2ili/gui/gui_export.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 6cc9b206..acd8d72e 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -655,7 +655,9 @@ def action_do_export(): # try: # add logger info to check what selection is used - logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") + #18.12.2024 + #logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") + logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids(False) - without automatic extension of neigbouring wwn elements}") if emodel == "VSA_KEK_2019_LV95": logger.info("Start Exporting VSA_KEK_2019_LV95") diff --git a/qgepqwat2ili/gui/gui_export.py b/qgepqwat2ili/gui/gui_export.py index 739899e2..cf755928 100644 --- a/qgepqwat2ili/gui/gui_export.py +++ b/qgepqwat2ili/gui/gui_export.py @@ -208,15 +208,16 @@ def logs_next_to_file(self): return self.save_logs_next_to_file_checkbox.isChecked() @property - def selected_ids(self): + def selected_ids(self, selection_extend=true): if self.limit_checkbox.isChecked(): ids = [] for struct in self.structures: ids.append(str(struct["wn_obj_id"])) for reach in self.reaches: ids.append(str(reach["obj_id"])) - ids.append(str(reach["rp_from_fk_wastewater_networkelement"])) - ids.append(str(reach["rp_to_fk_wastewater_networkelement"])) + if selection_extend: + ids.append(str(reach["rp_from_fk_wastewater_networkelement"])) + ids.append(str(reach["rp_to_fk_wastewater_networkelement"])) return ids else: return None From 5cd4530054f851ac70c047e76c2e44c78ae0c98b Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:05:10 +0100 Subject: [PATCH 232/245] correction --- qgepqwat2ili/gui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index acd8d72e..092b6423 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -657,7 +657,7 @@ def action_do_export(): # add logger info to check what selection is used #18.12.2024 #logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") - logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids(False) - without automatic extension of neigbouring wwn elements}") + logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids(False)} - without automatic extension of neigbouring wwn elements") if emodel == "VSA_KEK_2019_LV95": logger.info("Start Exporting VSA_KEK_2019_LV95") From 863f733afd0460aeee1384d05515a54fa5cfec81 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:07:05 +0100 Subject: [PATCH 233/245] missing ) --- qgepqwat2ili/gui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 092b6423..1d6cbf6d 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -657,7 +657,7 @@ def action_do_export(): # add logger info to check what selection is used #18.12.2024 #logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") - logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids(False)} - without automatic extension of neigbouring wwn elements") + logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids(False))} - without automatic extension of neigbouring wwn elements") if emodel == "VSA_KEK_2019_LV95": logger.info("Start Exporting VSA_KEK_2019_LV95") From 90839248a703b23d58d4f6f7e69b8d591b06272a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:07:20 +0000 Subject: [PATCH 234/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/gui/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 1d6cbf6d..b6877a75 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -655,9 +655,11 @@ def action_do_export(): # try: # add logger info to check what selection is used - #18.12.2024 - #logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") - logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids(False))} - without automatic extension of neigbouring wwn elements") + # 18.12.2024 + # logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") + logger.info( + f"Start Exporting with selection {str(export_dialog.selected_ids(False))} - without automatic extension of neigbouring wwn elements" + ) if emodel == "VSA_KEK_2019_LV95": logger.info("Start Exporting VSA_KEK_2019_LV95") From cf0b7ebcff54ba94774899a71d7af58817abb071 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:10:00 +0100 Subject: [PATCH 235/245] correct True --- qgepqwat2ili/gui/gui_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/gui/gui_export.py b/qgepqwat2ili/gui/gui_export.py index cf755928..5adaf2a8 100644 --- a/qgepqwat2ili/gui/gui_export.py +++ b/qgepqwat2ili/gui/gui_export.py @@ -208,7 +208,7 @@ def logs_next_to_file(self): return self.save_logs_next_to_file_checkbox.isChecked() @property - def selected_ids(self, selection_extend=true): + def selected_ids(self, selection_extend=True): if self.limit_checkbox.isChecked(): ids = [] for struct in self.structures: From e808daa408cf8accc1ec7186ca3e4767a5cfd53c Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:34:00 +0100 Subject: [PATCH 236/245] adapt selected_ids / improve qgep_export_utils --- qgepqwat2ili/gui/__init__.py | 2 +- qgepqwat2ili/gui/gui_export.py | 4 ++- qgepqwat2ili/utils/qgep_export_utils.py | 36 ++++++++++++------------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index b6877a75..4148c76b 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -658,7 +658,7 @@ def action_do_export(): # 18.12.2024 # logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") logger.info( - f"Start Exporting with selection {str(export_dialog.selected_ids(False))} - without automatic extension of neigbouring wwn elements" + f"Start Exporting with selection {str(export_dialog.selected_ids)} - without automatic extension of neigbouring wwn elements" ) if emodel == "VSA_KEK_2019_LV95": diff --git a/qgepqwat2ili/gui/gui_export.py b/qgepqwat2ili/gui/gui_export.py index 5adaf2a8..8bcb2c9c 100644 --- a/qgepqwat2ili/gui/gui_export.py +++ b/qgepqwat2ili/gui/gui_export.py @@ -208,7 +208,9 @@ def logs_next_to_file(self): return self.save_logs_next_to_file_checkbox.isChecked() @property - def selected_ids(self, selection_extend=True): + def selected_ids(self): + selection_extend = False + if self.limit_checkbox.isChecked(): ids = [] for struct in self.structures: diff --git a/qgepqwat2ili/utils/qgep_export_utils.py b/qgepqwat2ili/utils/qgep_export_utils.py index e29d29a8..60571184 100644 --- a/qgepqwat2ili/utils/qgep_export_utils.py +++ b/qgepqwat2ili/utils/qgep_export_utils.py @@ -1388,17 +1388,17 @@ def get_connected_we_to_re(subset_reaches): # 10.12.2024 -def get_connected_overflow_to_wn_ids(selected_ids): +def get_connected_overflow_to_wn_ids(selected_ids_ov): """ Get all connected wastewater_nodes from overflows.fk_overflow_to """ - if selected_ids is None: + if selected_ids_ov is None: return None - if not selected_ids: + if not selected_ids_ov: return None else: logger.debug( - f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids_ov} ..." ) connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) @@ -1406,7 +1406,7 @@ def get_connected_overflow_to_wn_ids(selected_ids): connected_overflow_to_wn_ids = [] - subset_text = get_selection_text_for_in_statement(selected_ids) + subset_text = get_selection_text_for_in_statement(selected_ids_ov) # select all connected to wastewater_nodes from provided subset of reaches cursor.execute( @@ -1580,15 +1580,15 @@ def get_ws_selected_ww_networkelements(selected_wwn): # 10.1.2024 -def filter_reaches(selected_ids): +def filter_reaches(selected_ids_to_filter): """ - Filter out reaches from selected_ids + Filter out reaches from selected_ids_to_filter """ - if selected_ids is None: + if selected_ids_to_filter is None: return None else: - logger.debug(f"Filter out reaches from selected_ids {selected_ids} ...") + logger.debug(f"Filter out reaches from selected_ids {selected_ids_to_filter} ...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) cursor = connection.cursor() @@ -1596,7 +1596,7 @@ def filter_reaches(selected_ids): subset_reaches_ids = [] all_reaches_ids = [] - get_selection_text_for_in_statement(selected_ids) + get_selection_text_for_in_statement(selected_ids_to_filter) # select all reaches cursor.execute("SELECT obj_id FROM qgep_od.reach;") @@ -1621,7 +1621,7 @@ def filter_reaches(selected_ids): all_reaches_ids.append(strrow) # logger.debug(f" building up '{all_reaches_ids}' ...") - for list_item in selected_ids: + for list_item in selected_ids_to_filter: if list_item in all_reaches_ids: subset_reaches_ids.append(list_item) logger.debug( @@ -1655,20 +1655,20 @@ def remove_from_selection(selected_ids, remove_ids): return selected_ids -def add_to_selection(selected_ids, add_ids): +def add_to_selection(selected_ids2, add_ids): """ Append ids to selected_ids """ - if selected_ids is None: + if selected_ids2 is None: return None if add_ids is None: - return selected_ids + return selected_ids2 else: - if selected_ids is None: - selected_ids = [] + if selected_ids2 is None: + selected_ids2 = [] for list_item in add_ids: # selected_ids = selected_ids.append(list_item) - selected_ids.append(list_item) + selected_ids2.append(list_item) - return selected_ids + return selected_ids2 From 966ab60a9f958aec440e490774d894d6ce320bc1 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:44:04 +0100 Subject: [PATCH 237/245] backup subset_ids --- qgepqwat2ili/qgepsia405/export.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 20c9829f..0867a17f 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -63,6 +63,9 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # Logging for debugging logger.debug(f"print subset_ids: '{str(subset_ids)}'") + # make a backup copy of subset_id - as it is beeing changed - don't know why + subset_ids_original = subset_ids + subset_wws_ids = [] if filtered: @@ -87,6 +90,11 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids = subset_ids_original + logger.debug( + f"reprint subset_ids: {subset_ids}", + ) + subset_ids_reaches = [] subset_ids_reaches = filter_reaches(subset_ids) logger.debug( From 589cf1f4ae99cc657684588db61f83795763ef43 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:44:24 +0000 Subject: [PATCH 238/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/gui/gui_export.py | 2 +- qgepqwat2ili/qgepsia405/export.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/gui/gui_export.py b/qgepqwat2ili/gui/gui_export.py index 8bcb2c9c..935d9e27 100644 --- a/qgepqwat2ili/gui/gui_export.py +++ b/qgepqwat2ili/gui/gui_export.py @@ -210,7 +210,7 @@ def logs_next_to_file(self): @property def selected_ids(self): selection_extend = False - + if self.limit_checkbox.isChecked(): ids = [] for struct in self.structures: diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 0867a17f..5e3efadf 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -65,7 +65,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske # make a backup copy of subset_id - as it is beeing changed - don't know why subset_ids_original = subset_ids - + subset_wws_ids = [] if filtered: @@ -94,7 +94,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.debug( f"reprint subset_ids: {subset_ids}", ) - + subset_ids_reaches = [] subset_ids_reaches = filter_reaches(subset_ids) logger.debug( From 79040df3610c4697a1b3757b6902265a1e8550a2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Wed, 18 Dec 2024 22:53:30 +0100 Subject: [PATCH 239/245] subset_ids_original testing --- qgepqwat2ili/qgepsia405/export.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 0867a17f..7ab92f71 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -64,7 +64,8 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.debug(f"print subset_ids: '{str(subset_ids)}'") # make a backup copy of subset_id - as it is beeing changed - don't know why - subset_ids_original = subset_ids + subset_ids_original = selection if selection is not None else [] + logger.debug(f"print subset_ids_original: '{str(subset_ids_original)}'") subset_wws_ids = [] @@ -90,6 +91,10 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + + logger.debug( + f"reprint subset_ids_original: {subset_ids_original}", + ) subset_ids = subset_ids_original logger.debug( f"reprint subset_ids: {subset_ids}", From cdba8548ecb91022cbad394bf75cffbad5e2045f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:59:45 +0000 Subject: [PATCH 240/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 923c2f93..3cd2a0e3 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -67,7 +67,6 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske subset_ids_original = selection if selection is not None else [] logger.debug(f"print subset_ids_original: '{str(subset_ids_original)}'") - subset_wws_ids = [] From 3638cde12b6da8872931990094cd56c417c1ac4a Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 19 Dec 2024 08:32:11 +0100 Subject: [PATCH 241/245] set variables back to selection_extend = True, as current solution --- qgepqwat2ili/gui/__init__.py | 8 ++++---- qgepqwat2ili/gui/gui_export.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 4148c76b..55720b05 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -656,10 +656,10 @@ def action_do_export(): # add logger info to check what selection is used # 18.12.2024 - # logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") - logger.info( - f"Start Exporting with selection {str(export_dialog.selected_ids)} - without automatic extension of neigbouring wwn elements" - ) + logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") + # logger.info( + # f"Start Exporting with selection {str(export_dialog.selected_ids)} - without automatic extension of neigbouring wwn elements" + # ) if emodel == "VSA_KEK_2019_LV95": logger.info("Start Exporting VSA_KEK_2019_LV95") diff --git a/qgepqwat2ili/gui/gui_export.py b/qgepqwat2ili/gui/gui_export.py index 935d9e27..dc090bbe 100644 --- a/qgepqwat2ili/gui/gui_export.py +++ b/qgepqwat2ili/gui/gui_export.py @@ -209,7 +209,8 @@ def logs_next_to_file(self): @property def selected_ids(self): - selection_extend = False + # Variable to choose if automatic selection_extend or not + selection_extend = True if self.limit_checkbox.isChecked(): ids = [] From da232d4889f19638f5f7f18573beed8a202e4627 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 07:40:06 +0000 Subject: [PATCH 242/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/gui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 55720b05..0febaf36 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -658,7 +658,7 @@ def action_do_export(): # 18.12.2024 logger.info(f"Start Exporting with selection {str(export_dialog.selected_ids)}") # logger.info( - # f"Start Exporting with selection {str(export_dialog.selected_ids)} - without automatic extension of neigbouring wwn elements" + # f"Start Exporting with selection {str(export_dialog.selected_ids)} - without automatic extension of neigbouring wwn elements" # ) if emodel == "VSA_KEK_2019_LV95": From 05a8e52599d6766c06fa4aa8862da9998c2ccf78 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 19 Dec 2024 08:47:09 +0100 Subject: [PATCH 243/245] update export kek --- qgepqwat2ili/qgep/export.py | 174 +++++++++++++++++++++++++++--------- 1 file changed, 130 insertions(+), 44 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index cf838817..283f4db4 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -56,7 +56,19 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e ws_off_sia405abwasser = False # 1. Filtering - check if selection filtered = selection is not None + + # Logging for debugging + logger.debug(f"print filtered '{str(filtered)}'") + subset_ids = selection if selection is not None else [] + # Logging for debugging + logger.debug(f"print subset_ids: '{str(subset_ids)}'") + + # make a backup copy of subset_id - as it is beeing changed - don't know why + + subset_ids_original = selection if selection is not None else [] + logger.debug(f"print subset_ids_original: '{str(subset_ids_original)}'") + subset_wws_ids = [] if filtered: @@ -81,6 +93,15 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e f"5 + 2 + 3 + 4 adapted_subset_ids: {adapted_subset_ids}", ) # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + + logger.debug( + f"reprint subset_ids_original: {subset_ids_original}", + ) + subset_ids = subset_ids_original + logger.debug( + f"reprint subset_ids: {subset_ids}", + ) + subset_ids_reaches = [] subset_ids_reaches = filter_reaches(subset_ids) logger.debug( @@ -313,55 +334,120 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e logger.info("Exporting QGEP.pipe_profile -> ABWASSER.rohrprofil, ABWASSER.metaattribute") qgep_export_utils.export_pipe_profile() - logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") - qgep_export_utils.export_reach_point() - - logger.info( - "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" - ) - query = qgep_session.query(qgep_model.wastewater_node) - if filtered: - query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # add sql statement to logger - statement = query.statement - logger.debug(f" selection query = {statement}") - for row in query: - # AVAILABLE FIELDS IN QGEP.wastewater_node - - # --- wastewater_networkelement --- - # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark - - # --- wastewater_node --- + # with or without check_fk_in_subset + # if filtered + if filtered or ws_off_sia405abwasser: + logger.info( + "Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute" + ) + qgep_export_utils.export_reach_point_check_fk_in_subset() - # --- _bwrel_ --- - # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + logger.info( + "Exporting QGEP.wastewater_node (check_fk_in_subset) -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + ) + # cannot be moved to qgep_export_utils because fk_hydr_geometry is only in VSA-DSS but not in SIA405 Abwasser and KEK + # qgep_export_utils.export_wastewater_node_check_fk_in_subset() + + query = qgep_session.query(qgep_model.wastewater_node) + if filtered: + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + # **qgep_export_utils.wastewater_networkelement_common(row), + **qgep_export_utils.wastewater_networkelement_common_check_fk_in_subset(row), + # --- abwasserknoten --- + # TODO : WARNING : fk_hydr_geometry is not mapped + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() - # --- _rel_ --- - # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + logger.info( + "Exporting QGEP.reach (check_fk_in_subset) -> ABWASSER.haltung, ABWASSER.metaattribute" + ) + qgep_export_utils.export_reach_check_fk_in_subset() - # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + # not filtered and not ws_off_sia405abwasser + else: + logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + qgep_export_utils.export_reach_point() - abwasserknoten = abwasser_model.abwasserknoten( - # FIELDS TO MAP TO ABWASSER.abwasserknoten - # --- baseclass --- - # --- sia405_baseclass --- - **qgep_export_utils.base_common(row, "abwasserknoten"), - # --- abwassernetzelement --- - **qgep_export_utils.wastewater_networkelement_common(row), - # --- abwasserknoten --- - # TODO : WARNING : fk_hydr_geometry is not mapped - lage=ST_Force2D(row.situation_geometry), - rueckstaukote=row.backflow_level, - sohlenkote=row.bottom_level, - ) - abwasser_session.add(abwasserknoten) - qgep_export_utils.create_metaattributes(row) - print(".", end="") - logger.info("done") - abwasser_session.flush() + logger.info( + "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + ) + # qgep_export_utils.export_wastewater_node() + + query = qgep_session.query(qgep_model.wastewater_node) + if filtered: + query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = abwasser_model.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **qgep_export_utils.base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **qgep_export_utils.wastewater_networkelement_common(row), + # --- abwasserknoten --- + # TODO : WARNING : fk_hydr_geometry is not mapped + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + qgep_export_utils.create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() - logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") - qgep_export_utils.export_reach() + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + qgep_export_utils.export_reach() if ws_off_sia405abwasser: logger.info( From 6f57ae0bf2a72314ddc2de797d178e183611257d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 20 Dec 2024 15:08:23 +0100 Subject: [PATCH 244/245] adapt cases export_reach_point and export_reach_point_check_fk_in_subset --- qgepqwat2ili/qgepsia405/export.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 3cd2a0e3..9f15e81a 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -334,12 +334,24 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske qgep_export_utils.export_pipe_profile() # with or without check_fk_in_subset - # if filtered - if filtered or ws_off_sia405abwasser: - logger.info( - "Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute" + if (filtered or ws_off_sia405abwasser): + logger.debug( + f"Filtered = {str(filtered)} and ws_off_sia405abwasser = {str(ws_off_sia405abwasser)}" ) - qgep_export_utils.export_reach_point_check_fk_in_subset() + # new 20.12.24 + # subset_id exists + if filtered: + logger.info( + "Exporting QGEP.reach_point (check_fk_in_subset) -> ABWASSER.haltungspunkt, ABWASSER.metaattribute" + ) + qgep_export_utils.export_reach_point_check_fk_in_subset() + # subset_id = [] + else: + logger.info( + "Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute" + ) + qgep_export_utils.export_reach_point() + logger.info( "Exporting QGEP.wastewater_node (check_fk_in_subset) -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" From 59d05a1f763212373f21b769009490ac5b103cf8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:08:41 +0000 Subject: [PATCH 245/245] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 9f15e81a..52c5730c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -334,7 +334,7 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske qgep_export_utils.export_pipe_profile() # with or without check_fk_in_subset - if (filtered or ws_off_sia405abwasser): + if filtered or ws_off_sia405abwasser: logger.debug( f"Filtered = {str(filtered)} and ws_off_sia405abwasser = {str(ws_off_sia405abwasser)}" ) @@ -352,7 +352,6 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske ) qgep_export_utils.export_reach_point() - logger.info( "Exporting QGEP.wastewater_node (check_fk_in_subset) -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" )