From 715e199eca8c7369de16929189660ddf6c423ea0 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:21:30 +0200 Subject: [PATCH 01/55] add new check def check_fk_owner_null() --- qgepqwat2ili/utils/ili2db.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 3313c4d6..06706dff 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -172,6 +172,43 @@ def check_identifier_null(): return identifier_null_check +# Checking if MAMDATORY eigentuemerref not is Null +def check_fk_owner_null(): + + logger.info("INTEGRITY CHECK missing owner references fk_owner...") + print("INTEGRITY CHECK missing owner references fk_owner...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + missing_fk_owner_count = 0 + for notsubclass in [ + # SIA405 Abwasser + ("wastewater_structure"), + ]: + cursor.execute( + f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_owner_count} WHERE fk_owner is null;" + ) + # use cursor.fetchone()[0] instead of cursor.rowcount + logger.info( + f"Number of datasets in {notsubclass} without fk_owner : {cursor.fetchone()[0]}" + ) + + if cursor.fetchone() is None: + missing_fk_owner_count = missing_fk_owner_count + else: + missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) + + if missing_fk_owner_count == 0: + check_fk_owner_null = True + logger.info("OK: all mandatory fk_owner set in qgep_od!") + else: + missing_fk_owner_count = False + logger.info(f"ERROR: Missing mandatory fk_owner in qgep_od: {missing_fk_owner_count}") + print(f"ERROR: Missing mandatory fk_owner: {missing_fk_owner_count}") + + return check_fk_owner_null def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From 43adf7de5be96d73e330683e6890c079d0f3c216 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:23:48 +0200 Subject: [PATCH 02/55] def check_fk_operator_null --- qgepqwat2ili/utils/ili2db.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 06706dff..6d04735e 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -210,6 +210,44 @@ def check_fk_owner_null(): return check_fk_owner_null +# Checking if MAMDATORY eigentuemerref not is Null +def check_fk_operator_null(): + + logger.info("INTEGRITY CHECK missing operator references fk_operator...") + print("INTEGRITY CHECK missing operator references fk_operator...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + missing_fk_operator_count = 0 + for notsubclass in [ + # SIA405 Abwasser + ("wastewater_structure"), + ]: + cursor.execute( + f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_operator_count} WHERE fk_operator is null;" + ) + # use cursor.fetchone()[0] instead of cursor.rowcount + logger.info( + f"Number of datasets in {notsubclass} without fk_operator : {cursor.fetchone()[0]}" + ) + + if cursor.fetchone() is None: + missing_fk_operator_count = missing_fk_operator_count + else: + missing_fk_operator_count = missing_fk_operator_count + int(cursor.fetchone()[0]) + + if missing_fk_operator_count == 0: + check_fk_operator_null = True + logger.info("OK: all mandatory fk_operator set in qgep_od!") + else: + missing_fk_operator_count = False + logger.info(f"ERROR: Missing mandatory fk_operator in qgep_od: {missing_fk_operator_count}") + print(f"ERROR: Missing mandatory fk_operator: {missing_fk_operator_count}") + + return check_fk_operator_null + def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From c6f52204aec659d6219c9f237b4b677de5fe5889 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:24:15 +0000 Subject: [PATCH 03/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/ili2db.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 6d04735e..cb580c28 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -172,6 +172,7 @@ def check_identifier_null(): return identifier_null_check + # Checking if MAMDATORY eigentuemerref not is Null def check_fk_owner_null(): @@ -210,6 +211,7 @@ def check_fk_owner_null(): return check_fk_owner_null + # Checking if MAMDATORY eigentuemerref not is Null def check_fk_operator_null(): @@ -243,11 +245,14 @@ def check_fk_operator_null(): logger.info("OK: all mandatory fk_operator set in qgep_od!") else: missing_fk_operator_count = False - logger.info(f"ERROR: Missing mandatory fk_operator in qgep_od: {missing_fk_operator_count}") + logger.info( + f"ERROR: Missing mandatory fk_operator in qgep_od: {missing_fk_operator_count}" + ) print(f"ERROR: Missing mandatory fk_operator: {missing_fk_operator_count}") return check_fk_operator_null - + + def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From 037eababc216ba98fd4e7fa86e7f4253735c72c1 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:28:28 +0200 Subject: [PATCH 04/55] check_fk_dataowner --- qgepqwat2ili/utils/ili2db.py | 87 +++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 6d04735e..1ffec935 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -175,8 +175,8 @@ def check_identifier_null(): # Checking if MAMDATORY eigentuemerref not is Null def check_fk_owner_null(): - logger.info("INTEGRITY CHECK missing owner references fk_owner...") - print("INTEGRITY CHECK missing owner references fk_owner...") + logger.info("INTEGRITY CHECK missing MAMDATORY owner references fk_owner...") + print("INTEGRITY CHECK missing MAMDATORY owner references fk_owner...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) @@ -213,8 +213,8 @@ def check_fk_owner_null(): # Checking if MAMDATORY eigentuemerref not is Null def check_fk_operator_null(): - logger.info("INTEGRITY CHECK missing operator references fk_operator...") - print("INTEGRITY CHECK missing operator references fk_operator...") + logger.info("INTEGRITY CHECK missing MAMDATORY operator references fk_operator...") + print("INTEGRITY CHECK missing MAMDATORY operator references fk_operator...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) @@ -247,7 +247,84 @@ def check_fk_operator_null(): print(f"ERROR: Missing mandatory fk_operator: {missing_fk_operator_count}") return check_fk_operator_null - + +# Checking if MAMDATORY datenherrref not is Null +def check_fk_dataowner_null(): + + logger.info("INTEGRITY CHECK missing dataowner references fk_dataowner...") + print("INTEGRITY CHECK missing dataowner references fk_dataowner...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + missing_fk_dataowner_count = 0 + for notsubclass in [ + # VSA-KEK + ("file"), + ("data_media"), + ("maintenance_event"), + # SIA405 Abwasser + ("organisation"), + ("wastewater_structure"), + ("wastewater_networkelement"), + ("structure_part"), + ("reach_point"), + ("pipe_profile"), + # VSA-DSS + ("catchment_area"), + ("connection_object"), + ("control_center"), + ("hazard_source"), + ("hydr_geometry"), + ("hydraulic_char_data"), + ("measurement_result"), + ("measurement_series"), + ("measuring_device"), + ("measuring_point"), + ("mechanical_pretreatment"), + ("overflow"), + ("overflow_char"), + ("retention_body"), + ("river_bank"), + ("river_bed"), + ("sector_water_body"), + ("substance"), + ("surface_runoff_parameters"), + ("surface_water_bodies"), + ("throttle_shut_off_unit"), + ("waste_water_treatment"), + ("water_catchment"), + ("water_control_structure"), + ("water_course_segment"), + ("wwtp_energy_use"), + ("zone"), + ]: + cursor.execute( + f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_dataowner_count} WHERE fk_dataowner is null;" + ) + # use cursor.fetchone()[0] instead of cursor.rowcount + logger.info( + f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" + ) + + if cursor.fetchone() is None: + missing_fk_dataowner_count = missing_fk_dataowner_count + else: + missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) + + if missing_fk_dataowner_count == 0: + check_fk_dataowner_null = True + logger.info("OK: all mandatory fk_dataowner set in qgep_od!") + else: + missing_fk_dataowner_count = False + logger.info(f"ERROR: Missing mandatory fk_dataowner in qgep_od: {missing_fk_dataowner_count}") + print(f"ERROR: Missing mandatory fk_dataowner: {missing_fk_dataowner_count}") + + return check_fk_dataowner_null + + + def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From 8702b5c39cd9191e2e853e2ed7e0bcadde5c4b31 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:30:45 +0200 Subject: [PATCH 05/55] check_fk_provider --- qgepqwat2ili/utils/ili2db.py | 73 ++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 1ffec935..4100a871 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -323,7 +323,80 @@ def check_fk_dataowner_null(): return check_fk_dataowner_null +# Checking if MAMDATORY datenlieferantref not is Null +def check_fk_provider_null(): + logger.info("INTEGRITY CHECK missing provider references fk_provider...") + print("INTEGRITY CHECK missing provider references fk_provider...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + missing_fk_provider_count = 0 + for notsubclass in [ + # VSA-KEK + ("file"), + ("data_media"), + ("maintenance_event"), + # SIA405 Abwasser + ("organisation"), + ("wastewater_structure"), + ("wastewater_networkelement"), + ("structure_part"), + ("reach_point"), + ("pipe_profile"), + # VSA-DSS + ("catchment_area"), + ("connection_object"), + ("control_center"), + ("hazard_source"), + ("hydr_geometry"), + ("hydraulic_char_data"), + ("measurement_result"), + ("measurement_series"), + ("measuring_device"), + ("measuring_point"), + ("mechanical_pretreatment"), + ("overflow"), + ("overflow_char"), + ("retention_body"), + ("river_bank"), + ("river_bed"), + ("sector_water_body"), + ("substance"), + ("surface_runoff_parameters"), + ("surface_water_bodies"), + ("throttle_shut_off_unit"), + ("waste_water_treatment"), + ("water_catchment"), + ("water_control_structure"), + ("water_course_segment"), + ("wwtp_energy_use"), + ("zone"), + ]: + cursor.execute( + f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_provider_count} WHERE fk_provider is null;" + ) + # use cursor.fetchone()[0] instead of cursor.rowcount + logger.info( + f"Number of datasets in {notsubclass} without fk_provider : {cursor.fetchone()[0]}" + ) + + if cursor.fetchone() is None: + missing_fk_provider_count = missing_fk_provider_count + else: + missing_fk_provider_count = missing_fk_provider_count + int(cursor.fetchone()[0]) + + if missing_fk_provider_count == 0: + check_fk_provider_null = True + logger.info("OK: all mandatory fk_provider set in qgep_od!") + else: + missing_fk_provider_count = False + logger.info(f"ERROR: Missing mandatory fk_provider in qgep_od: {missing_fk_provider_count}") + print(f"ERROR: Missing mandatory fk_provider: {missing_fk_provider_count}") + + return check_fk_provider_null def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From fe4fbbf4b9fa77fd284cbe6cade6a37da212257e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:31:00 +0000 Subject: [PATCH 06/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/ili2db.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 4100a871..c8b4294a 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -172,6 +172,7 @@ def check_identifier_null(): return identifier_null_check + # Checking if MAMDATORY eigentuemerref not is Null def check_fk_owner_null(): @@ -210,6 +211,7 @@ def check_fk_owner_null(): return check_fk_owner_null + # Checking if MAMDATORY eigentuemerref not is Null def check_fk_operator_null(): @@ -243,11 +245,14 @@ def check_fk_operator_null(): logger.info("OK: all mandatory fk_operator set in qgep_od!") else: missing_fk_operator_count = False - logger.info(f"ERROR: Missing mandatory fk_operator in qgep_od: {missing_fk_operator_count}") + logger.info( + f"ERROR: Missing mandatory fk_operator in qgep_od: {missing_fk_operator_count}" + ) print(f"ERROR: Missing mandatory fk_operator: {missing_fk_operator_count}") return check_fk_operator_null + # Checking if MAMDATORY datenherrref not is Null def check_fk_dataowner_null(): @@ -318,11 +323,14 @@ def check_fk_dataowner_null(): logger.info("OK: all mandatory fk_dataowner set in qgep_od!") else: missing_fk_dataowner_count = False - logger.info(f"ERROR: Missing mandatory fk_dataowner in qgep_od: {missing_fk_dataowner_count}") + logger.info( + f"ERROR: Missing mandatory fk_dataowner in qgep_od: {missing_fk_dataowner_count}" + ) print(f"ERROR: Missing mandatory fk_dataowner: {missing_fk_dataowner_count}") return check_fk_dataowner_null + # Checking if MAMDATORY datenlieferantref not is Null def check_fk_provider_null(): @@ -393,11 +401,14 @@ def check_fk_provider_null(): logger.info("OK: all mandatory fk_provider set in qgep_od!") else: missing_fk_provider_count = False - logger.info(f"ERROR: Missing mandatory fk_provider in qgep_od: {missing_fk_provider_count}") + logger.info( + f"ERROR: Missing mandatory fk_provider in qgep_od: {missing_fk_provider_count}" + ) print(f"ERROR: Missing mandatory fk_provider: {missing_fk_provider_count}") return check_fk_provider_null + def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From 273ff72ae5131997de1972a2e37cf120f6bc8b35 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:34:48 +0200 Subject: [PATCH 07/55] add check_fk_owner_null --- qgepqwat2ili/gui/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index dc54c3d6..cb62b4bd 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -483,6 +483,28 @@ def action_do_export(): # just show hint, but continue # return + # 4. identifier check check_fk_owner_null + check_fk_owner_null = False + check_fk_owner_null = check_fk_owner_null() + if check_fk_owner_null: + print("OK: Integrity checks fk_owner not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_owner not isNull", + None, + ) + + else: + progress_dialog.close() + print("INFO: missing MANDATORY fk_owner") + show_hint( + "INFO: Missing MANDATORY fk_owner in schema qgep_od", + "Add missing MANDATORY fk_owner to get a valid INTERLIS export file. See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return + # Prepare the temporary ili2pg model progress_dialog.setLabelText("Creating ili schema..." + emodel) From 094bdb4c367059cd64966619047fa1d7ffd9e822 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:41:28 +0200 Subject: [PATCH 08/55] add check_fk_operator_null --- qgepqwat2ili/gui/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index cb62b4bd..a0360626 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -505,6 +505,28 @@ def action_do_export(): # just show hint, but continue # return + # 5. identifier check check_fk_operator_null + check_fk_operator_null = False + check_fk_operator_null = check_fk_operator_null() + if check_fk_operator_null: + print("OK: Integrity checks fk_operator not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_operator not isNull", + None, + ) + + else: + progress_dialog.close() + print("INFO: missing MANDATORY fk_operator") + show_hint( + "INFO: Missing MANDATORY fk_operator in schema qgep_od", + "Add missing MANDATORY fk_operator to get a valid INTERLIS export file. See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return + # Prepare the temporary ili2pg model progress_dialog.setLabelText("Creating ili schema..." + emodel) From e7dfea9ef97d67edae50409bb52c2ca9867c4294 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:46:19 +0200 Subject: [PATCH 09/55] add check_fk_dataowner_null --- qgepqwat2ili/gui/__init__.py | 51 +++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index a0360626..90d00d27 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -483,7 +483,7 @@ def action_do_export(): # just show hint, but continue # return - # 4. identifier check check_fk_owner_null + # 4. relation check check_fk_owner_null check_fk_owner_null = False check_fk_owner_null = check_fk_owner_null() if check_fk_owner_null: @@ -505,7 +505,7 @@ def action_do_export(): # just show hint, but continue # return - # 5. identifier check check_fk_operator_null + # 5. relation check check_fk_operator_null check_fk_operator_null = False check_fk_operator_null = check_fk_operator_null() if check_fk_operator_null: @@ -526,7 +526,52 @@ def action_do_export(): ) # just show hint, but continue # return - + + # 6. relation check check_fk_dataowner_null + check_fk_dataowner_null = False + check_fk_dataowner_null = check_fk_dataowner_null() + if check_fk_dataowner_null: + print("OK: Integrity checks fk_dataowner not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_dataowner not isNull", + None, + ) + + else: + progress_dialog.close() + print("INFO: missing fk_dataowner") + show_hint( + "INFO: Missing fk_dataowner in schema qgep_od", + "Add missing fk_dataowner to get a valid INTERLIS export file when converting to Release 2020 (will bei MANDATORY). See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return + + # 6. relation check check_fk_provider_null + check_fk_provider_null = False + check_fk_provider_null = check_fk_provider_null() + if check_fk_provider_null: + print("OK: Integrity checks fk_provider not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_provider not isNull", + None, + ) + + else: + progress_dialog.close() + print("INFO: missing fk_provider") + show_hint( + "INFO: Missing fk_provider in schema qgep_od", + "Add missing fk_provider to get a valid INTERLIS export file when converting to Release 2020 (will bei MANDATORY). See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return + + # Prepare the temporary ili2pg model progress_dialog.setLabelText("Creating ili schema..." + emodel) From ac9bc33f2013474b711635d38ec172b5323e38a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:48:50 +0000 Subject: [PATCH 10/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/gui/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 90d00d27..462e7e77 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -549,7 +549,7 @@ def action_do_export(): # just show hint, but continue # return - # 6. relation check check_fk_provider_null + # 6. relation check check_fk_provider_null check_fk_provider_null = False check_fk_provider_null = check_fk_provider_null() if check_fk_provider_null: @@ -571,7 +571,6 @@ def action_do_export(): # just show hint, but continue # return - # Prepare the temporary ili2pg model progress_dialog.setLabelText("Creating ili schema..." + emodel) From a35dd9a19b83ea54978a3d29ce96f18dffe63a66 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:49:42 +0200 Subject: [PATCH 11/55] correction --- qgepqwat2ili/utils/ili2db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index c8b4294a..b619856f 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -212,7 +212,7 @@ def check_fk_owner_null(): return check_fk_owner_null -# Checking if MAMDATORY eigentuemerref not is Null +# Checking if MAMDATORY betreiberref not is Null def check_fk_operator_null(): logger.info("INTEGRITY CHECK missing MAMDATORY operator references fk_operator...") From 47c728b82dbd4aea01523515f2e66e6809072c31 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 3 Oct 2024 16:56:18 +0200 Subject: [PATCH 12/55] add if flag_test: --- qgepqwat2ili/gui/__init__.py | 201 ++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 99 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 462e7e77..2d82345c 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -462,114 +462,117 @@ def action_do_export(): return # 3. identifier check check_identifier_null - check_identifier = False - check_identifier = check_identifier_null() - if check_identifier: - print("OK: Integrity checks identifiers not isNull") - show_success( - "Sucess", - "OK: Integrity checks identifiers not isNull", - None, - ) + if flag_test: + check_identifier = False + check_identifier = check_identifier_null() + if check_identifier: + print("OK: Integrity checks identifiers not isNull") + show_success( + "Sucess", + "OK: Integrity checks identifiers not isNull", + None, + ) - else: - progress_dialog.close() - print("INFO: missing identifiers") - show_hint( - "INFO: Missing identifiers in schema qgep_od", - "Add missing identifiers to get a valid INTERLIS export file. See qgep logs tab for details.", - None, - ) - # just show hint, but continue - # return + else: + progress_dialog.close() + print("INFO: missing identifiers") + show_hint( + "INFO: Missing identifiers in schema qgep_od", + "Add missing identifiers to get a valid INTERLIS export file. See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return # 4. relation check check_fk_owner_null - check_fk_owner_null = False - check_fk_owner_null = check_fk_owner_null() - if check_fk_owner_null: - print("OK: Integrity checks fk_owner not isNull") - show_success( - "Sucess", - "OK: Integrity checks fk_owner not isNull", - None, - ) + if flag_test: + check_fk_owner_null = False + check_fk_owner_null = check_fk_owner_null() + if check_fk_owner_null: + print("OK: Integrity checks fk_owner not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_owner not isNull", + None, + ) - else: - progress_dialog.close() - print("INFO: missing MANDATORY fk_owner") - show_hint( - "INFO: Missing MANDATORY fk_owner in schema qgep_od", - "Add missing MANDATORY fk_owner to get a valid INTERLIS export file. See qgep logs tab for details.", - None, - ) - # just show hint, but continue - # return - - # 5. relation check check_fk_operator_null - check_fk_operator_null = False - check_fk_operator_null = check_fk_operator_null() - if check_fk_operator_null: - print("OK: Integrity checks fk_operator not isNull") - show_success( - "Sucess", - "OK: Integrity checks fk_operator not isNull", - None, - ) + else: + progress_dialog.close() + print("INFO: missing MANDATORY fk_owner") + show_hint( + "INFO: Missing MANDATORY fk_owner in schema qgep_od", + "Add missing MANDATORY fk_owner to get a valid INTERLIS export file. See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return + + # 5. relation check check_fk_operator_null + check_fk_operator_null = False + check_fk_operator_null = check_fk_operator_null() + if check_fk_operator_null: + print("OK: Integrity checks fk_operator not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_operator not isNull", + None, + ) - else: - progress_dialog.close() - print("INFO: missing MANDATORY fk_operator") - show_hint( - "INFO: Missing MANDATORY fk_operator in schema qgep_od", - "Add missing MANDATORY fk_operator to get a valid INTERLIS export file. See qgep logs tab for details.", - None, - ) - # just show hint, but continue - # return + else: + progress_dialog.close() + print("INFO: missing MANDATORY fk_operator") + show_hint( + "INFO: Missing MANDATORY fk_operator in schema qgep_od", + "Add missing MANDATORY fk_operator to get a valid INTERLIS export file. See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return # 6. relation check check_fk_dataowner_null - check_fk_dataowner_null = False - check_fk_dataowner_null = check_fk_dataowner_null() - if check_fk_dataowner_null: - print("OK: Integrity checks fk_dataowner not isNull") - show_success( - "Sucess", - "OK: Integrity checks fk_dataowner not isNull", - None, - ) + if flag_test: + check_fk_dataowner_null = False + check_fk_dataowner_null = check_fk_dataowner_null() + if check_fk_dataowner_null: + print("OK: Integrity checks fk_dataowner not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_dataowner not isNull", + None, + ) - else: - progress_dialog.close() - print("INFO: missing fk_dataowner") - show_hint( - "INFO: Missing fk_dataowner in schema qgep_od", - "Add missing fk_dataowner to get a valid INTERLIS export file when converting to Release 2020 (will bei MANDATORY). See qgep logs tab for details.", - None, - ) - # just show hint, but continue - # return - - # 6. relation check check_fk_provider_null - check_fk_provider_null = False - check_fk_provider_null = check_fk_provider_null() - if check_fk_provider_null: - print("OK: Integrity checks fk_provider not isNull") - show_success( - "Sucess", - "OK: Integrity checks fk_provider not isNull", - None, - ) + else: + progress_dialog.close() + print("INFO: missing fk_dataowner") + show_hint( + "INFO: Missing fk_dataowner in schema qgep_od", + "Add missing fk_dataowner to get a valid INTERLIS export file when converting to Release 2020 (will bei MANDATORY). See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return + + # 6. relation check check_fk_provider_null + check_fk_provider_null = False + check_fk_provider_null = check_fk_provider_null() + if check_fk_provider_null: + print("OK: Integrity checks fk_provider not isNull") + show_success( + "Sucess", + "OK: Integrity checks fk_provider not isNull", + None, + ) - else: - progress_dialog.close() - print("INFO: missing fk_provider") - show_hint( - "INFO: Missing fk_provider in schema qgep_od", - "Add missing fk_provider to get a valid INTERLIS export file when converting to Release 2020 (will bei MANDATORY). See qgep logs tab for details.", - None, - ) - # just show hint, but continue - # return + else: + progress_dialog.close() + print("INFO: missing fk_provider") + show_hint( + "INFO: Missing fk_provider in schema qgep_od", + "Add missing fk_provider to get a valid INTERLIS export file when converting to Release 2020 (will bei MANDATORY). See qgep logs tab for details.", + None, + ) + # just show hint, but continue + # return # Prepare the temporary ili2pg model progress_dialog.setLabelText("Creating ili schema..." + emodel) From f1670be57a9e8ab1a546c0eba0cd428d6fdcb547 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 11:27:18 +0200 Subject: [PATCH 13/55] add new functions --- qgepqwat2ili/gui/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 2d82345c..0d73516d 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -29,6 +29,10 @@ from ..utils.ili2db import ( # neu 22.7.2022; get_xtf_model,; neu 31.3.2023; neu 12.4.2023 check_identifier_null, check_organisation_subclass_data, + check_fk_owner_null, + check_fk_operator_null, + check_fk_dataowner_null, + check_fk_provider_null, check_wastewater_structure_subclass_data, create_ili_schema, export_xtf_data, From 284c8dd835f61c690b39c5de1d4c3254f8ecb12c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:28:20 +0000 Subject: [PATCH 14/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/gui/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 0d73516d..addc5eef 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -27,12 +27,12 @@ from ..qgepsia405.export import qgep_export as qgepsia405_export from ..qgepsia405.import_ import qgep_import as qgepsia405_import from ..utils.ili2db import ( # neu 22.7.2022; get_xtf_model,; neu 31.3.2023; neu 12.4.2023 - check_identifier_null, - check_organisation_subclass_data, - check_fk_owner_null, - check_fk_operator_null, check_fk_dataowner_null, + check_fk_operator_null, + check_fk_owner_null, check_fk_provider_null, + check_identifier_null, + check_organisation_subclass_data, check_wastewater_structure_subclass_data, create_ili_schema, export_xtf_data, From 6b25d3f490cf75028bcdd61fcd6419b8d8ea60e6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 11:29:04 +0200 Subject: [PATCH 15/55] take out time.sleep --- qgepqwat2ili/gui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 0d73516d..63471c97 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -102,7 +102,7 @@ def action_do_importc(): progress_dialog.show() progress_dialog.setLabelText("waiting...") # delays the execution for 5.5 secs. - time.sleep(5.5) + # time.sleep(5.5) progress_dialog.close # end action_do_importc From f4c06ce76e2accd5748ce705bb1727145ed1a07d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 11:29:48 +0200 Subject: [PATCH 16/55] take out time.sleep --- qgepqwat2ili/gui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 267d7c37..eb834d1d 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -360,7 +360,7 @@ def action_import(plugin): # 31.5.2024 should not be needed anymore # progress_dialog.setLabelText("Set main_cover manually after import if vw_qgep_wastewater_structure does not display correctly!") - time.sleep(2) + # time.sleep(2) # to add option to run main_cover.sql manually - see postimport.py # 24.7.2022 / moved to end From 6effa7e3fc4bfb6f8d35c0c40312f068601b39ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:30:58 +0000 Subject: [PATCH 17/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/gui/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index eb834d1d..70c55401 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -3,7 +3,6 @@ import tempfile # 12.7.2022 for testing import time -import time import webbrowser from types import SimpleNamespace From 5e18a10c124f94e0b85b18355c1e509a61dcd0f5 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 11:45:43 +0200 Subject: [PATCH 18/55] correct wrong select statements --- qgepqwat2ili/utils/ili2db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index b619856f..363ff799 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -189,7 +189,7 @@ def check_fk_owner_null(): ("wastewater_structure"), ]: cursor.execute( - f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_owner_count} WHERE fk_owner is null;" + f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_owner is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount logger.info( @@ -306,7 +306,7 @@ def check_fk_dataowner_null(): ("zone"), ]: cursor.execute( - f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_dataowner_count} WHERE fk_dataowner is null;" + f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_dataowner is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount logger.info( @@ -384,7 +384,7 @@ def check_fk_provider_null(): ("zone"), ]: cursor.execute( - f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_provider_count} WHERE fk_provider is null;" + f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_provider is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount logger.info( From b1b6f7a4023e75f74c8a4f37422ee358e6fa26f6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:46:10 +0000 Subject: [PATCH 19/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/ili2db.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 363ff799..45c5b85b 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -188,9 +188,7 @@ def check_fk_owner_null(): # SIA405 Abwasser ("wastewater_structure"), ]: - cursor.execute( - f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_owner is null;" - ) + cursor.execute(f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_owner is null;") # use cursor.fetchone()[0] instead of cursor.rowcount logger.info( f"Number of datasets in {notsubclass} without fk_owner : {cursor.fetchone()[0]}" From 3b0159bfe4495ce27932c518f31150dffd39eca5 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 11:49:10 +0200 Subject: [PATCH 20/55] correct wrong variable names --- qgepqwat2ili/gui/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 70c55401..42acb245 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -489,9 +489,9 @@ def action_do_export(): # 4. relation check check_fk_owner_null if flag_test: - check_fk_owner_null = False - check_fk_owner_null = check_fk_owner_null() - if check_fk_owner_null: + check_fk_owner = False + check_fk_owner = check_fk_owner_null() + if check_fk_owner: print("OK: Integrity checks fk_owner not isNull") show_success( "Sucess", @@ -511,9 +511,9 @@ def action_do_export(): # return # 5. relation check check_fk_operator_null - check_fk_operator_null = False - check_fk_operator_null = check_fk_operator_null() - if check_fk_operator_null: + check_fk_operator = False + check_fk_operator = check_fk_operator_null() + if check_fk_operator: print("OK: Integrity checks fk_operator not isNull") show_success( "Sucess", @@ -534,9 +534,9 @@ def action_do_export(): # 6. relation check check_fk_dataowner_null if flag_test: - check_fk_dataowner_null = False - check_fk_dataowner_null = check_fk_dataowner_null() - if check_fk_dataowner_null: + check_fk_dataowner = False + check_fk_dataowner = check_fk_dataowner_null() + if check_fk_dataowner: print("OK: Integrity checks fk_dataowner not isNull") show_success( "Sucess", @@ -556,9 +556,9 @@ def action_do_export(): # return # 6. relation check check_fk_provider_null - check_fk_provider_null = False - check_fk_provider_null = check_fk_provider_null() - if check_fk_provider_null: + check_fk_provider = False + check_fk_provider = check_fk_provider_null() + if check_fk_provider: print("OK: Integrity checks fk_provider not isNull") show_success( "Sucess", From c5bb4b6b32ed5ab8469d5755c0ff74998f9f9a44 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 11:57:07 +0200 Subject: [PATCH 21/55] correct select statement --- qgepqwat2ili/utils/ili2db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 45c5b85b..a9f0c297 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -226,7 +226,7 @@ def check_fk_operator_null(): ("wastewater_structure"), ]: cursor.execute( - f"SELECT COUNT(obj_id) FROM qgep_od.{missing_fk_operator_count} WHERE fk_operator is null;" + f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_operator is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount logger.info( From d97d102a5229ccab8fcc818c7d9729e339a78fb7 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 14:19:54 +0200 Subject: [PATCH 22/55] correct wrong else statment / testing --- qgepqwat2ili/utils/ili2db.py | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index a9f0c297..82fdf299 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -199,11 +199,16 @@ def check_fk_owner_null(): else: missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) + # add for testing + logger.info( + f"missing_fk_owner_count : {missing_fk_owner_count}" + ) + if missing_fk_owner_count == 0: check_fk_owner_null = True logger.info("OK: all mandatory fk_owner set in qgep_od!") else: - missing_fk_owner_count = False + check_fk_owner_null = False logger.info(f"ERROR: Missing mandatory fk_owner in qgep_od: {missing_fk_owner_count}") print(f"ERROR: Missing mandatory fk_owner: {missing_fk_owner_count}") @@ -237,12 +242,16 @@ def check_fk_operator_null(): missing_fk_operator_count = missing_fk_operator_count else: missing_fk_operator_count = missing_fk_operator_count + int(cursor.fetchone()[0]) + # add for testing + logger.info( + f"missing_fk_operator_count : {missing_fk_operator_count}" + ) if missing_fk_operator_count == 0: check_fk_operator_null = True logger.info("OK: all mandatory fk_operator set in qgep_od!") else: - missing_fk_operator_count = False + check_fk_operator_null = False logger.info( f"ERROR: Missing mandatory fk_operator in qgep_od: {missing_fk_operator_count}" ) @@ -307,20 +316,34 @@ def check_fk_dataowner_null(): f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_dataowner is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount + + # hilfvariable + tttt = int(cursor.fetchone()[0]) + + #logger.info( + # f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" + #) logger.info( - f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" + f"2. Number of datasets in {notsubclass} without fk_dataowner (tttt) : {tttt}" ) - if cursor.fetchone() is None: + # if cursor.fetchone() is None: + if tttt == 0: missing_fk_dataowner_count = missing_fk_dataowner_count else: - missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) + #missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) + missing_fk_dataowner_count = missing_fk_dataowner_count + tttt + + # add for testing + logger.info( + f"missing_fk_dataowner_count : {missing_fk_dataowner_count}" + ) if missing_fk_dataowner_count == 0: check_fk_dataowner_null = True logger.info("OK: all mandatory fk_dataowner set in qgep_od!") else: - missing_fk_dataowner_count = False + check_fk_dataowner_null = False logger.info( f"ERROR: Missing mandatory fk_dataowner in qgep_od: {missing_fk_dataowner_count}" ) @@ -389,16 +412,35 @@ def check_fk_provider_null(): f"Number of datasets in {notsubclass} without fk_provider : {cursor.fetchone()[0]}" ) + # if no data in class cursor will be None if cursor.fetchone() is None: missing_fk_provider_count = missing_fk_provider_count + # add for testing + logger.info( + f"No data in class {notsubclass}" + ) + #logger.info( + # f"cursor.fetchone() is None : {int(cursor.fetchone()[0])}" + #) else: - missing_fk_provider_count = missing_fk_provider_count + int(cursor.fetchone()[0]) + if int(cursor.fetchone()[0]) == 0: + # missing_fk_provider_count = missing_fk_provider_count + logger.info( + f"{(cursor.fetchone()[0])} missing fk_provider in class {notsubclass}" + ) + else: + missing_fk_provider_count = missing_fk_provider_count + int(cursor.fetchone()[0]) + + # add for testing + logger.info( + f"missing_fk_provider_count : {missing_fk_provider_count}" + ) if missing_fk_provider_count == 0: check_fk_provider_null = True logger.info("OK: all mandatory fk_provider set in qgep_od!") else: - missing_fk_provider_count = False + check_fk_provider_null = False logger.info( f"ERROR: Missing mandatory fk_provider in qgep_od: {missing_fk_provider_count}" ) From 501da3f910ba5d38c955d5f2dcfcf1215adb5ae1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:21:50 +0000 Subject: [PATCH 23/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/ili2db.py | 52 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 82fdf299..bb26d374 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -199,10 +199,8 @@ def check_fk_owner_null(): else: missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) - # add for testing - logger.info( - f"missing_fk_owner_count : {missing_fk_owner_count}" - ) + # add for testing + logger.info(f"missing_fk_owner_count : {missing_fk_owner_count}") if missing_fk_owner_count == 0: check_fk_owner_null = True @@ -242,10 +240,8 @@ def check_fk_operator_null(): missing_fk_operator_count = missing_fk_operator_count else: missing_fk_operator_count = missing_fk_operator_count + int(cursor.fetchone()[0]) - # add for testing - logger.info( - f"missing_fk_operator_count : {missing_fk_operator_count}" - ) + # add for testing + logger.info(f"missing_fk_operator_count : {missing_fk_operator_count}") if missing_fk_operator_count == 0: check_fk_operator_null = True @@ -316,28 +312,24 @@ def check_fk_dataowner_null(): f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_dataowner is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount - + # hilfvariable tttt = int(cursor.fetchone()[0]) - - #logger.info( + + # logger.info( # f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" - #) - logger.info( - f"2. Number of datasets in {notsubclass} without fk_dataowner (tttt) : {tttt}" - ) + # ) + logger.info(f"2. Number of datasets in {notsubclass} without fk_dataowner (tttt) : {tttt}") # if cursor.fetchone() is None: if tttt == 0: missing_fk_dataowner_count = missing_fk_dataowner_count else: - #missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) + # missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) missing_fk_dataowner_count = missing_fk_dataowner_count + tttt - # add for testing - logger.info( - f"missing_fk_dataowner_count : {missing_fk_dataowner_count}" - ) + # add for testing + logger.info(f"missing_fk_dataowner_count : {missing_fk_dataowner_count}") if missing_fk_dataowner_count == 0: check_fk_dataowner_null = True @@ -415,26 +407,20 @@ def check_fk_provider_null(): # if no data in class cursor will be None if cursor.fetchone() is None: missing_fk_provider_count = missing_fk_provider_count - # add for testing - logger.info( - f"No data in class {notsubclass}" - ) - #logger.info( + # add for testing + logger.info(f"No data in class {notsubclass}") + # logger.info( # f"cursor.fetchone() is None : {int(cursor.fetchone()[0])}" - #) + # ) else: if int(cursor.fetchone()[0]) == 0: # missing_fk_provider_count = missing_fk_provider_count - logger.info( - f"{(cursor.fetchone()[0])} missing fk_provider in class {notsubclass}" - ) + logger.info(f"{(cursor.fetchone()[0])} missing fk_provider in class {notsubclass}") else: missing_fk_provider_count = missing_fk_provider_count + int(cursor.fetchone()[0]) - # add for testing - logger.info( - f"missing_fk_provider_count : {missing_fk_provider_count}" - ) + # add for testing + logger.info(f"missing_fk_provider_count : {missing_fk_provider_count}") if missing_fk_provider_count == 0: check_fk_provider_null = True From 86ed2c98a6c4719805b3c4d0a732ef9b222f2c3c Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 14:55:45 +0200 Subject: [PATCH 24/55] adapt corrections to all fk checks --- qgepqwat2ili/utils/ili2db.py | 74 +++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 82fdf299..d301f854 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -108,6 +108,7 @@ def check_identifier_null(): cursor = connection.cursor() missing_identifier_count = 0 + # add classes to be checked for notsubclass in [ # VSA-KEK ("file"), @@ -153,14 +154,23 @@ def check_identifier_null(): f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE identifier is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount + # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + class_identifier_count = int(cursor.fetchone()[0]) logger.info( - f"Number of datasets in {notsubclass} without identifier : {cursor.fetchone()[0]}" + f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_identifier_count}" ) - - if cursor.fetchone() is None: + + #if cursor.fetchone() is None: + if class_identifier_count == 0: missing_identifier_count = missing_identifier_count else: - missing_identifier_count = missing_identifier_count + int(cursor.fetchone()[0]) + #missing_identifier_count = missing_identifier_count + int(cursor.fetchone()[0]) + missing_identifier_count = missing_identifier_count + class_identifier_count(cursor.fetchone()[0]) + + # add for testing + logger.info( + f"missing_fk_dataowner_count : {missing_identifier_count}" + ) if missing_identifier_count == 0: identifier_null_check = True @@ -184,20 +194,28 @@ def check_fk_owner_null(): cursor = connection.cursor() missing_fk_owner_count = 0 + # add MANDATORY classes to be checked for notsubclass in [ # SIA405 Abwasser ("wastewater_structure"), ]: cursor.execute(f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_owner is null;") # use cursor.fetchone()[0] instead of cursor.rowcount + # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + class_fk_owner_count = int(cursor.fetchone()[0]) + #logger.info( + # f"Number of datasets in {notsubclass} without fk_owner : {cursor.fetchone()[0]}" + #) logger.info( - f"Number of datasets in {notsubclass} without fk_owner : {cursor.fetchone()[0]}" + f"Number of datasets in {notsubclass} without fk_owner (tttt) : {class_fk_owner_count}" ) - if cursor.fetchone() is None: + #if cursor.fetchone() is None: + if class_fk_owner_count == 0: missing_fk_owner_count = missing_fk_owner_count else: - missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) + #missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) + missing_fk_owner_count = missing_fk_owner_count + class_fk_owner_count # add for testing logger.info( @@ -226,6 +244,8 @@ def check_fk_operator_null(): cursor = connection.cursor() missing_fk_operator_count = 0 + + # add MANDATORY classes to be checked for notsubclass in [ # SIA405 Abwasser ("wastewater_structure"), @@ -271,6 +291,7 @@ def check_fk_dataowner_null(): cursor = connection.cursor() missing_fk_dataowner_count = 0 + # add MANDATORY classes to be checked for notsubclass in [ # VSA-KEK ("file"), @@ -316,23 +337,22 @@ def check_fk_dataowner_null(): f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_dataowner is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount - - # hilfvariable - tttt = int(cursor.fetchone()[0]) + # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + class_fk_dataowner_count = int(cursor.fetchone()[0]) #logger.info( # f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" #) logger.info( - f"2. Number of datasets in {notsubclass} without fk_dataowner (tttt) : {tttt}" + f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_fk_dataowner_count}" ) # if cursor.fetchone() is None: - if tttt == 0: + if class_fk_dataowner_count == 0: missing_fk_dataowner_count = missing_fk_dataowner_count else: #missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) - missing_fk_dataowner_count = missing_fk_dataowner_count + tttt + missing_fk_dataowner_count = missing_fk_dataowner_count + class_fk_dataowner_count # add for testing logger.info( @@ -363,6 +383,7 @@ def check_fk_provider_null(): cursor = connection.cursor() missing_fk_provider_count = 0 + # add MANDATORY classes to be checked for notsubclass in [ # VSA-KEK ("file"), @@ -408,28 +429,21 @@ def check_fk_provider_null(): f"SELECT COUNT(obj_id) FROM qgep_od.{notsubclass} WHERE fk_provider is null;" ) # use cursor.fetchone()[0] instead of cursor.rowcount + # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + class_fk_provider_count = int(cursor.fetchone()[0]) + #logger.info( + # f"Number of datasets in {notsubclass} without fk_provider : {cursor.fetchone()[0]}" + #) logger.info( - f"Number of datasets in {notsubclass} without fk_provider : {cursor.fetchone()[0]}" + f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_fk_provider_count}" ) - # if no data in class cursor will be None - if cursor.fetchone() is None: + #if cursor.fetchone() is None: + if class_fk_provider_count == 0: missing_fk_provider_count = missing_fk_provider_count - # add for testing - logger.info( - f"No data in class {notsubclass}" - ) - #logger.info( - # f"cursor.fetchone() is None : {int(cursor.fetchone()[0])}" - #) else: - if int(cursor.fetchone()[0]) == 0: - # missing_fk_provider_count = missing_fk_provider_count - logger.info( - f"{(cursor.fetchone()[0])} missing fk_provider in class {notsubclass}" - ) - else: - missing_fk_provider_count = missing_fk_provider_count + int(cursor.fetchone()[0]) + # missing_fk_provider_count = missing_fk_provider_count + missing_fk_provider_count = missing_fk_provider_count + class_fk_provider_count # add for testing logger.info( From 98dcf98f566a8fb74a7cb430eadffe19eb3f120c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:57:48 +0000 Subject: [PATCH 25/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/ili2db.py | 39 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index b6cb2c65..292e156e 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -159,18 +159,18 @@ def check_identifier_null(): logger.info( f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_identifier_count}" ) - - #if cursor.fetchone() is None: + + # if cursor.fetchone() is None: if class_identifier_count == 0: missing_identifier_count = missing_identifier_count else: - #missing_identifier_count = missing_identifier_count + int(cursor.fetchone()[0]) - missing_identifier_count = missing_identifier_count + class_identifier_count(cursor.fetchone()[0]) + # missing_identifier_count = missing_identifier_count + int(cursor.fetchone()[0]) + missing_identifier_count = missing_identifier_count + class_identifier_count( + cursor.fetchone()[0] + ) - # add for testing - logger.info( - f"missing_fk_dataowner_count : {missing_identifier_count}" - ) + # add for testing + logger.info(f"missing_fk_dataowner_count : {missing_identifier_count}") if missing_identifier_count == 0: identifier_null_check = True @@ -203,18 +203,18 @@ def check_fk_owner_null(): # use cursor.fetchone()[0] instead of cursor.rowcount # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ class_fk_owner_count = int(cursor.fetchone()[0]) - #logger.info( + # logger.info( # f"Number of datasets in {notsubclass} without fk_owner : {cursor.fetchone()[0]}" - #) + # ) logger.info( f"Number of datasets in {notsubclass} without fk_owner (tttt) : {class_fk_owner_count}" ) - #if cursor.fetchone() is None: + # if cursor.fetchone() is None: if class_fk_owner_count == 0: missing_fk_owner_count = missing_fk_owner_count else: - #missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) + # missing_fk_owner_count = missing_fk_owner_count + int(cursor.fetchone()[0]) missing_fk_owner_count = missing_fk_owner_count + class_fk_owner_count # add for testing @@ -335,20 +335,19 @@ def check_fk_dataowner_null(): # use cursor.fetchone()[0] instead of cursor.rowcount # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ class_fk_dataowner_count = int(cursor.fetchone()[0]) - - #logger.info( + + # logger.info( # f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" - #) + # ) logger.info( f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_fk_dataowner_count}" ) - # if cursor.fetchone() is None: if class_fk_dataowner_count == 0: missing_fk_dataowner_count = missing_fk_dataowner_count else: - #missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) + # missing_fk_dataowner_count = missing_fk_dataowner_count + int(cursor.fetchone()[0]) missing_fk_dataowner_count = missing_fk_dataowner_count + class_fk_dataowner_count # add for testing @@ -426,14 +425,14 @@ def check_fk_provider_null(): # use cursor.fetchone()[0] instead of cursor.rowcount # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ class_fk_provider_count = int(cursor.fetchone()[0]) - #logger.info( + # logger.info( # f"Number of datasets in {notsubclass} without fk_provider : {cursor.fetchone()[0]}" - #) + # ) logger.info( f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_fk_provider_count}" ) - #if cursor.fetchone() is None: + # if cursor.fetchone() is None: if class_fk_provider_count == 0: missing_fk_provider_count = missing_fk_provider_count else: From ec0ed08e4ad6f0acfd1490ea22ba022d97e74d0f Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 15:03:09 +0200 Subject: [PATCH 26/55] make check_fk_operator and fk_owner mandatory, else we will get errors anyway --- qgepqwat2ili/gui/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 42acb245..2e7a30f5 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -507,8 +507,8 @@ def action_do_export(): "Add missing MANDATORY fk_owner to get a valid INTERLIS export file. See qgep logs tab for details.", None, ) - # just show hint, but continue - # return + # make mandatory + return # 5. relation check check_fk_operator_null check_fk_operator = False @@ -529,8 +529,8 @@ def action_do_export(): "Add missing MANDATORY fk_operator to get a valid INTERLIS export file. See qgep logs tab for details.", None, ) - # just show hint, but continue - # return + # make mandatory + return # 6. relation check check_fk_dataowner_null if flag_test: From cb96bc993f6b833cb2e288fb44a6e7e633384f5a Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 15:07:26 +0200 Subject: [PATCH 27/55] adapt messages to ERROR --- qgepqwat2ili/gui/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 2e7a30f5..936d3909 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -501,9 +501,9 @@ def action_do_export(): else: progress_dialog.close() - print("INFO: missing MANDATORY fk_owner") + print("ERROR: missing MANDATORY fk_owner") show_hint( - "INFO: Missing MANDATORY fk_owner in schema qgep_od", + "ERROR: Missing MANDATORY fk_owner in schema qgep_od", "Add missing MANDATORY fk_owner to get a valid INTERLIS export file. See qgep logs tab for details.", None, ) @@ -523,9 +523,9 @@ def action_do_export(): else: progress_dialog.close() - print("INFO: missing MANDATORY fk_operator") + print("ERROR: missing MANDATORY fk_operator") show_hint( - "INFO: Missing MANDATORY fk_operator in schema qgep_od", + "ERROR: Missing MANDATORY fk_operator in schema qgep_od", "Add missing MANDATORY fk_operator to get a valid INTERLIS export file. See qgep logs tab for details.", None, ) From defeda584922d5bee14c2d439aaf04ca1791bbe0 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 15:35:51 +0200 Subject: [PATCH 28/55] adapt error message to be failure --- qgepqwat2ili/gui/__init__.py | 4 ++-- qgepqwat2ili/utils/ili2db.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/qgepqwat2ili/gui/__init__.py b/qgepqwat2ili/gui/__init__.py index 936d3909..df7d0b9e 100644 --- a/qgepqwat2ili/gui/__init__.py +++ b/qgepqwat2ili/gui/__init__.py @@ -502,7 +502,7 @@ def action_do_export(): else: progress_dialog.close() print("ERROR: missing MANDATORY fk_owner") - show_hint( + show_failure( "ERROR: Missing MANDATORY fk_owner in schema qgep_od", "Add missing MANDATORY fk_owner to get a valid INTERLIS export file. See qgep logs tab for details.", None, @@ -524,7 +524,7 @@ def action_do_export(): else: progress_dialog.close() print("ERROR: missing MANDATORY fk_operator") - show_hint( + show_failure( "ERROR: Missing MANDATORY fk_operator in schema qgep_od", "Add missing MANDATORY fk_operator to get a valid INTERLIS export file. See qgep logs tab for details.", None, diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 292e156e..a8e88090 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -157,7 +157,7 @@ def check_identifier_null(): # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ class_identifier_count = int(cursor.fetchone()[0]) logger.info( - f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_identifier_count}" + f"Number of datasets in class '{notsubclass}' without identifier : {class_identifier_count}" ) # if cursor.fetchone() is None: @@ -170,7 +170,7 @@ def check_identifier_null(): ) # add for testing - logger.info(f"missing_fk_dataowner_count : {missing_identifier_count}") + logger.info(f"missing_identifier_count : {missing_identifier_count}") if missing_identifier_count == 0: identifier_null_check = True @@ -204,10 +204,10 @@ def check_fk_owner_null(): # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ class_fk_owner_count = int(cursor.fetchone()[0]) # logger.info( - # f"Number of datasets in {notsubclass} without fk_owner : {cursor.fetchone()[0]}" + # f"Number of datasets in class '{notsubclass}' without fk_owner : {cursor.fetchone()[0]}" # ) logger.info( - f"Number of datasets in {notsubclass} without fk_owner (tttt) : {class_fk_owner_count}" + f"Number of datasets in class '{notsubclass}' without fk_owner : {class_fk_owner_count}" ) # if cursor.fetchone() is None: @@ -253,7 +253,7 @@ def check_fk_operator_null(): ) # use cursor.fetchone()[0] instead of cursor.rowcount logger.info( - f"Number of datasets in {notsubclass} without fk_operator : {cursor.fetchone()[0]}" + f"Number of datasets in class '{notsubclass}' without fk_operator : {cursor.fetchone()[0]}" ) if cursor.fetchone() is None: @@ -337,10 +337,10 @@ def check_fk_dataowner_null(): class_fk_dataowner_count = int(cursor.fetchone()[0]) # logger.info( - # f"Number of datasets in {notsubclass} without fk_dataowner : {cursor.fetchone()[0]}" + # f"Number of datasets in class '{notsubclass}' without fk_dataowner : {cursor.fetchone()[0]}" # ) logger.info( - f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_fk_dataowner_count}" + f"Number of datasets in class '{notsubclass}' without fk_dataowner : {class_fk_dataowner_count}" ) # if cursor.fetchone() is None: @@ -426,10 +426,10 @@ def check_fk_provider_null(): # add variable and store result of cursor.fetchone()[0] as the next call will give None value instead of count https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ class_fk_provider_count = int(cursor.fetchone()[0]) # logger.info( - # f"Number of datasets in {notsubclass} without fk_provider : {cursor.fetchone()[0]}" + # f"Number of datasets in class '{notsubclass}' without fk_provider : {cursor.fetchone()[0]}" # ) logger.info( - f"Number of datasets in {notsubclass} without fk_dataowner (tttt) : {class_fk_provider_count}" + f"Number of datasets in class '{notsubclass}' without fk_dataowner : {class_fk_provider_count}" ) # if cursor.fetchone() is None: From 4bcd24ccaef74f83b1a6e22576674d8cac6cabf4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 20:23:30 +0200 Subject: [PATCH 29/55] fix check_fk_in_subsetid --- qgepqwat2ili/qgepsia405/export.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 32c7bb82..17b3074c 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -130,10 +130,13 @@ def check_fk_in_subsetid(subset, relation): logger.info(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") return tid_maker.tid_for_row(relation) else: - logger.info( - f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" - ) - return None + if filtered: + logger.info( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" + ) + return None + else + return tid_maker.tid_for_row(relation) def create_metaattributes(row): metaattribute = ABWASSER.metaattribute( From 5929a6afb24acb8f2d1f44a4d8c0c009fc815b18 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 20:36:50 +0200 Subject: [PATCH 30/55] fix check_fk_in_subsetid in qgepdss export --- qgepqwat2ili/qgepdss/export.py | 11 +++++++---- qgepqwat2ili/qgepsia405/export.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index b4035693..08346b7f 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -140,10 +140,13 @@ def check_fk_in_subsetid(subset, relation): logger.info(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") return tid_maker.tid_for_row(relation) else: - logger.info( - f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" - ) - return None + if filtered: + logger.info( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" + ) + return None + else: + return tid_maker.tid_for_row(relation) def create_metaattributes(row): metaattribute = ABWASSER.metaattribute( diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 17b3074c..238e8927 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -135,7 +135,7 @@ def check_fk_in_subsetid(subset, relation): f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" ) return None - else + else: return tid_maker.tid_for_row(relation) def create_metaattributes(row): From 39ded51395871b50a1b274693a2cbc653bd4400f Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 20:37:51 +0200 Subject: [PATCH 31/55] fix check_fk_in_subsetid for qgep export --- qgepqwat2ili/qgep/export.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 06a14407..5589ea59 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -130,10 +130,13 @@ def check_fk_in_subsetid(subset, relation): logger.info(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") return tid_maker.tid_for_row(relation) else: - logger.info( - f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" - ) - return None + if filtered: + logger.info( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in subset - replaced with None instead!" + ) + return None + else: + return tid_maker.tid_for_row(relation) def create_metaattributes(row): metaattribute = ABWASSER.metaattribute( From 2b9dbb2c5666535a91ef2fa1c9da1b56901deb44 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:19:31 +0200 Subject: [PATCH 32/55] def get_wwtp_structure_ids() --- qgepqwat2ili/utils/ili2db.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index a8e88090..cb706ad6 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -436,7 +436,7 @@ def check_fk_provider_null(): if class_fk_provider_count == 0: missing_fk_provider_count = missing_fk_provider_count else: - # missing_fk_provider_count = missing_fk_provider_count + # missing_fk_provider_count = missing_fk_provider_count + int(cursor.fetchone()[0]) missing_fk_provider_count = missing_fk_provider_count + class_fk_provider_count # add for testing @@ -454,6 +454,32 @@ def check_fk_provider_null(): return check_fk_provider_null +def get_wwtp_structure_ids(): +# get list of id's of class wwtp_strucuture (ARABauwerk) + + logger.info("get list of id's of class wwtp_strucuture (ARABauwerk)...") + print("get list of id's of class wwtp_strucuture (ARABauwerk)...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + wwtp_structure_ids = [] + + cursor.execute( + f"select obj_id from qgep_od.wwtp_strucuture;" + ) + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + wwtp_structure_count = int(cursor.fetchone()[0]) + if wwtp_structure_count == 0: + wwtp_structure_ids = None + else: + records = cursor.fetchall() + for row in records: + wwtp_structure_ids = wwtp_structure_ids + row[0] + ',' + + return wwtp_structure_ids + def create_ili_schema(schema, model, log_path, recreate_schema=False): logger.info("CONNECTING TO DATABASE...") From 095f1855af66249db640d7d0c65b12bdd74e5519 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:20:04 +0000 Subject: [PATCH 33/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/utils/ili2db.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index cb706ad6..441d9051 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -454,8 +454,9 @@ def check_fk_provider_null(): return check_fk_provider_null + def get_wwtp_structure_ids(): -# get list of id's of class wwtp_strucuture (ARABauwerk) + # get list of id's of class wwtp_strucuture (ARABauwerk) logger.info("get list of id's of class wwtp_strucuture (ARABauwerk)...") print("get list of id's of class wwtp_strucuture (ARABauwerk)...") @@ -466,9 +467,7 @@ def get_wwtp_structure_ids(): wwtp_structure_ids = [] - cursor.execute( - f"select obj_id from qgep_od.wwtp_strucuture;" - ) + cursor.execute(f"select obj_id from qgep_od.wwtp_strucuture;") # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ wwtp_structure_count = int(cursor.fetchone()[0]) if wwtp_structure_count == 0: @@ -476,7 +475,7 @@ def get_wwtp_structure_ids(): else: records = cursor.fetchall() for row in records: - wwtp_structure_ids = wwtp_structure_ids + row[0] + ',' + wwtp_structure_ids = wwtp_structure_ids + row[0] + "," return wwtp_structure_ids From c55a24182e34da8d003ad0c3acd85b8248bc3386 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:27:23 +0200 Subject: [PATCH 34/55] add wwtp_structure_ids --- qgepqwat2ili/qgepsia405/export.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 238e8927..42507183 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -36,6 +36,12 @@ def qgep_export(selection=None, labels_file=None, orientation=None): filtered = selection is not None subset_ids = selection if selection is not None else [] + # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure + wwtp_structure_ids = get_wwtp_structure_ids() + logger.info( + "wwtp_structure_ids : {wwtp_structure_ids}", + ) + # Orientation oriented = orientation is not None if oriented: From 03c0c3c59c1bff894ab631d8760b01090b23f590 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:27:48 +0000 Subject: [PATCH 35/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 42507183..92d4a008 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -37,10 +37,10 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wwtp_structure_ids = get_wwtp_structure_ids() + get_wwtp_structure_ids() logger.info( - "wwtp_structure_ids : {wwtp_structure_ids}", - ) + "wwtp_structure_ids : {wwtp_structure_ids}", + ) # Orientation oriented = orientation is not None From 7fc2ddf1886bd24c4bff5f992b3867c976d258b2 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:34:54 +0200 Subject: [PATCH 36/55] from ..utils.ili2db import get_wwtp_structure_ids --- qgepqwat2ili/qgepsia405/export.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 42507183..5b3be0f2 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -7,6 +7,9 @@ from .. import utils from ..utils.various import logger +#4.10.2024 +from ..utils.ili2db import get_wwtp_structure_ids + from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model From 18db5062f133b5971645a52b936d4df58773994b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:35:34 +0000 Subject: [PATCH 37/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 01f1d2ec..02da7d7a 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -6,10 +6,10 @@ from sqlalchemy.sql import text from .. import utils -from ..utils.various import logger -#4.10.2024 -from ..utils.ili2db import get_wwtp_structure_ids +# 4.10.2024 +from ..utils.ili2db import get_wwtp_structure_ids +from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model From 7573aa8a35ce74968e5346e7a579fb0d6774fc7d Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:37:02 +0200 Subject: [PATCH 38/55] wwtp_structure_ids = get_wwtp_structure_ids() --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 01f1d2ec..78f25b90 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,7 +40,7 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - get_wwtp_structure_ids() + wwtp_structure_ids = get_wwtp_structure_ids() logger.info( "wwtp_structure_ids : {wwtp_structure_ids}", ) From 3fd608b947dcee9e081ca3c4c7d3a18cfb0304ce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:37:27 +0000 Subject: [PATCH 39/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index d28facd0..02da7d7a 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,7 +40,7 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wwtp_structure_ids = get_wwtp_structure_ids() + get_wwtp_structure_ids() logger.info( "wwtp_structure_ids : {wwtp_structure_ids}", ) From 243b9468205234ad3db51b506657add9f5cc106e Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:40:26 +0200 Subject: [PATCH 40/55] wwtp_structure_ids = get_wwtp_structure_ids() 2nd try --- qgepqwat2ili/qgepsia405/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 02da7d7a..342c7bb1 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,7 +40,8 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - get_wwtp_structure_ids() + wwtp_structure_ids = None + wwtp_structure_ids = get_wwtp_structure_ids() logger.info( "wwtp_structure_ids : {wwtp_structure_ids}", ) From ecd82b40a3229bd4e914ba8bcd7987d4b4cda8aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:40:40 +0000 Subject: [PATCH 41/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 342c7bb1..02da7d7a 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,8 +40,7 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wwtp_structure_ids = None - wwtp_structure_ids = get_wwtp_structure_ids() + get_wwtp_structure_ids() logger.info( "wwtp_structure_ids : {wwtp_structure_ids}", ) From 1e9b411aac0db1e62407f3a8c65ecbd1c19fba57 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:45:58 +0200 Subject: [PATCH 42/55] correct wwtp_strucuture to wwtp_structure --- qgepqwat2ili/qgepsia405/export.py | 5 +++-- qgepqwat2ili/utils/ili2db.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 02da7d7a..75ce3f3b 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -39,8 +39,9 @@ def qgep_export(selection=None, labels_file=None, orientation=None): filtered = selection is not None subset_ids = selection if selection is not None else [] - # get list of id's of class wwtp_strucuture (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - get_wwtp_structure_ids() + # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure + wwtp_structure_ids = None + wwtp_structure_ids = get_wwtp_structure_ids() logger.info( "wwtp_structure_ids : {wwtp_structure_ids}", ) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 441d9051..78d20e56 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -456,10 +456,10 @@ def check_fk_provider_null(): def get_wwtp_structure_ids(): - # get list of id's of class wwtp_strucuture (ARABauwerk) + # get list of id's of class wwtp_structure (ARABauwerk) - logger.info("get list of id's of class wwtp_strucuture (ARABauwerk)...") - print("get list of id's of class wwtp_strucuture (ARABauwerk)...") + logger.info("get list of id's of class wwtp_structure (ARABauwerk)...") + print("get list of id's of class wwtp_structure (ARABauwerk)...") connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) connection.set_session(autocommit=True) @@ -467,7 +467,7 @@ def get_wwtp_structure_ids(): wwtp_structure_ids = [] - cursor.execute(f"select obj_id from qgep_od.wwtp_strucuture;") + cursor.execute(f"select obj_id from qgep_od.wwtp_structure;") # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ wwtp_structure_count = int(cursor.fetchone()[0]) if wwtp_structure_count == 0: From 2d5523b458f3370b8ee011b950cb001cfd657c19 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:47:22 +0000 Subject: [PATCH 43/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index 75ce3f3b..d8803b17 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,8 +40,7 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wwtp_structure_ids = None - wwtp_structure_ids = get_wwtp_structure_ids() + get_wwtp_structure_ids() logger.info( "wwtp_structure_ids : {wwtp_structure_ids}", ) From bb69b21f71942b1d0794a7e6b3f594743c1ea329 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:55:27 +0200 Subject: [PATCH 44/55] adapt get_wwtp_structure_ids(): is not a count but select statement - therefore check for none --- qgepqwat2ili/utils/ili2db.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 78d20e56..5dcfce6e 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -467,10 +467,14 @@ def get_wwtp_structure_ids(): wwtp_structure_ids = [] - cursor.execute(f"select obj_id from qgep_od.wwtp_structure;") + # select all obj_id from wwtp_structure + cursor.execute( + f"SELECT obj_id FROM qgep_od.wwtp_structure;" + ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - wwtp_structure_count = int(cursor.fetchone()[0]) - if wwtp_structure_count == 0: + #wwtp_structure_count = int(cursor.fetchone()[0]) + #if wwtp_structure_count == 0: + if cursor.fetchone() is None: wwtp_structure_ids = None else: records = cursor.fetchall() From 1567cafd024b91d3c7520d585232cd9735c476a6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 21:56:27 +0200 Subject: [PATCH 45/55] change variable name --- qgepqwat2ili/qgepsia405/export.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index d8803b17..e8220b74 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,9 +40,10 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - get_wwtp_structure_ids() + wwtp_structure_id_list = None + wwtp_structure_id_list = get_wwtp_structure_ids() logger.info( - "wwtp_structure_ids : {wwtp_structure_ids}", + "wwtp_structure_id_list : {wwtp_structure_id_list}", ) # Orientation From 790c79c4eda062f0af7e94c5ad1ef51680c1e0a1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:56:43 +0000 Subject: [PATCH 46/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 3 +-- qgepqwat2ili/utils/ili2db.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index e8220b74..0d750ab8 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -40,8 +40,7 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wwtp_structure_id_list = None - wwtp_structure_id_list = get_wwtp_structure_ids() + get_wwtp_structure_ids() logger.info( "wwtp_structure_id_list : {wwtp_structure_id_list}", ) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 5dcfce6e..abbfdfcf 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -468,12 +468,10 @@ def get_wwtp_structure_ids(): wwtp_structure_ids = [] # select all obj_id from wwtp_structure - cursor.execute( - f"SELECT obj_id FROM qgep_od.wwtp_structure;" - ) + cursor.execute(f"SELECT obj_id FROM qgep_od.wwtp_structure;") # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ - #wwtp_structure_count = int(cursor.fetchone()[0]) - #if wwtp_structure_count == 0: + # wwtp_structure_count = int(cursor.fetchone()[0]) + # if wwtp_structure_count == 0: if cursor.fetchone() is None: wwtp_structure_ids = None else: From 28a27fe8c1c8e06cf0d8ada4d6a161570c7fd638 Mon Sep 17 00:00:00 2001 From: SJiB Date: Fri, 4 Oct 2024 23:43:22 +0200 Subject: [PATCH 47/55] rename wwtp_structure_id_list to wastewater_structure_id_sia405abwasser etc. - rewrite query --- qgepqwat2ili/qgepsia405/export.py | 12 +++++++----- qgepqwat2ili/utils/ili2db.py | 30 +++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index e8220b74..1e1e79f1 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -8,7 +8,7 @@ from .. import utils # 4.10.2024 -from ..utils.ili2db import get_wwtp_structure_ids +from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model @@ -40,12 +40,13 @@ def qgep_export(selection=None, labels_file=None, orientation=None): subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wwtp_structure_id_list = None - wwtp_structure_id_list = get_wwtp_structure_ids() + wastewater_structure_id_sia405abwasser_list = None + wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() logger.info( - "wwtp_structure_id_list : {wwtp_structure_id_list}", + f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", ) + # Orientation oriented = orientation is not None if oriented: @@ -218,7 +219,8 @@ def wastewater_networkelement_common(row): """ return { - "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid(wastewater_structure_id_sia405abwasser_list, row.fk_wastewater_structure__REL), "bemerkung": truncate(emptystr_to_null(row.remark), 80), "bezeichnung": null_to_emptystr(row.identifier), } diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 5dcfce6e..c8c40165 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -455,8 +455,8 @@ def check_fk_provider_null(): return check_fk_provider_null -def get_wwtp_structure_ids(): - # get list of id's of class wwtp_structure (ARABauwerk) +def skip_wwtp_structure_ids(): + # get list of id's of class wastewater_structure without wwtp_structure (ARABauwerk) logger.info("get list of id's of class wwtp_structure (ARABauwerk)...") print("get list of id's of class wwtp_structure (ARABauwerk)...") @@ -465,23 +465,35 @@ def get_wwtp_structure_ids(): connection.set_session(autocommit=True) cursor = connection.cursor() - wwtp_structure_ids = [] + not_wwtp_structure_ids = [] - # select all obj_id from wwtp_structure + # select all obj_id from wastewater_structure that are not in wwtp_structure cursor.execute( - f"SELECT obj_id FROM qgep_od.wwtp_structure;" + f"SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" ) + #remove - only for testing + #cursor.execute( + # f"SELECT * FROM qgep_od.organisation WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.private);" + #) + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ #wwtp_structure_count = int(cursor.fetchone()[0]) #if wwtp_structure_count == 0: if cursor.fetchone() is None: - wwtp_structure_ids = None + not_wwtp_structure_ids = None else: records = cursor.fetchall() for row in records: - wwtp_structure_ids = wwtp_structure_ids + row[0] + "," - - return wwtp_structure_ids + logger.info(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + # not_wwtp_structure_ids = not_wwtp_structure_ids + str(row[0]) + "," + strrow = str(row[0]) + #not_wwtp_structure_ids = ','.join([not_wwtp_structure_ids, strrow]) + #not_wwtp_structure_ids = not_wwtp_structure_ids + row[0] + not_wwtp_structure_ids.append(strrow) + logger.info(f" building up '{not_wwtp_structure_ids}' ...") + + return not_wwtp_structure_ids def create_ili_schema(schema, model, log_path, recreate_schema=False): From c6211127e652f69cb6f88157399ec723c82b3d2b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 21:45:13 +0000 Subject: [PATCH 48/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepsia405/export.py | 5 +++-- qgepqwat2ili/utils/ili2db.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index abedf4e3..e64f7a85 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -48,7 +48,6 @@ def qgep_export(selection=None, labels_file=None, orientation=None): f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", ) - # Orientation oriented = orientation is not None if oriented: @@ -222,7 +221,9 @@ def wastewater_networkelement_common(row): return { # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), - "abwasserbauwerkref": check_fk_in_subsetid(wastewater_structure_id_sia405abwasser_list, row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid( + wastewater_structure_id_sia405abwasser_list, row.fk_wastewater_structure__REL + ), "bemerkung": truncate(emptystr_to_null(row.remark), 80), "bezeichnung": null_to_emptystr(row.identifier), } diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 187c3e86..36fe6fb8 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -471,10 +471,10 @@ def skip_wwtp_structure_ids(): cursor.execute( f"SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" ) - #remove - only for testing - #cursor.execute( + # remove - only for testing + # cursor.execute( # f"SELECT * FROM qgep_od.organisation WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.private);" - #) + # ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ # wwtp_structure_count = int(cursor.fetchone()[0]) @@ -488,8 +488,8 @@ def skip_wwtp_structure_ids(): # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ # not_wwtp_structure_ids = not_wwtp_structure_ids + str(row[0]) + "," strrow = str(row[0]) - #not_wwtp_structure_ids = ','.join([not_wwtp_structure_ids, strrow]) - #not_wwtp_structure_ids = not_wwtp_structure_ids + row[0] + # not_wwtp_structure_ids = ','.join([not_wwtp_structure_ids, strrow]) + # not_wwtp_structure_ids = not_wwtp_structure_ids + row[0] not_wwtp_structure_ids.append(strrow) logger.info(f" building up '{not_wwtp_structure_ids}' ...") From 9ec46e717fb0157bd0573b33ddcea473bf937e59 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 10 Oct 2024 11:19:44 +0200 Subject: [PATCH 49/55] update qgep export.py --- qgepqwat2ili/qgep/export.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 5589ea59..8dd23d00 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -6,6 +6,8 @@ from sqlalchemy.sql import text from .. import utils +# 4.10.2024 +from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model @@ -36,6 +38,14 @@ def qgep_export(selection=None, labels_file=None, orientation=None): filtered = selection is not None subset_ids = selection if selection is not None else [] + # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure + + wastewater_structure_id_sia405abwasser_list = None + wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() + + logger.info( + f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", + ) # Orientation oriented = orientation is not None if oriented: @@ -211,7 +221,10 @@ def wastewater_networkelement_common(row): """ return { - "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid( + wastewater_structure_id_sia405abwasser_list, row.fk_wastewater_structure__REL + ), "bemerkung": truncate(emptystr_to_null(row.remark), 80), "bezeichnung": null_to_emptystr(row.identifier), } From 924bc15bb58da5d02c8ab0e78cedce87c0e8ecd4 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 10 Oct 2024 11:21:19 +0200 Subject: [PATCH 50/55] add empty lines --- qgepqwat2ili/qgep/export.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 8dd23d00..6f9a74ad 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -6,6 +6,7 @@ from sqlalchemy.sql import text from .. import utils + # 4.10.2024 from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.various import logger @@ -46,6 +47,7 @@ def qgep_export(selection=None, labels_file=None, orientation=None): logger.info( f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", ) + # Orientation oriented = orientation is not None if oriented: From 071b65de635981ae4d0b0b65a5c7bc749c185877 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 10 Oct 2024 11:28:41 +0200 Subject: [PATCH 51/55] update qgepdss export.py --- qgepqwat2ili/qgepdss/export.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 08346b7f..67c53595 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -6,6 +6,8 @@ from sqlalchemy.sql import text from .. import utils +# 4.10.2024 +from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model @@ -42,6 +44,21 @@ def qgep_export(selection=None, labels_file=None, orientation=None): # Logging for debugging logger.info(f"print subset_ids '{subset_ids}'") + + # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure + + + wastewater_structure_id_sia405abwasser_list = None + wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() + + logger.info( + f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", + ) + + + + + # Orientation oriented = orientation is not None From b5a76773d757d11e0f6d43d4cc856c3820cbe521 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:28:59 +0000 Subject: [PATCH 52/55] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- qgepqwat2ili/qgepdss/export.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 67c53595..7b0c845c 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -6,6 +6,7 @@ from sqlalchemy.sql import text from .. import utils + # 4.10.2024 from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.various import logger @@ -44,9 +45,8 @@ def qgep_export(selection=None, labels_file=None, orientation=None): # Logging for debugging logger.info(f"print subset_ids '{subset_ids}'") - - # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - + + # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure wastewater_structure_id_sia405abwasser_list = None wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() @@ -55,11 +55,6 @@ def qgep_export(selection=None, labels_file=None, orientation=None): f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", ) - - - - - # Orientation oriented = orientation is not None if oriented: From 56e1af4d0361c81bc932571ea5a885502a2b9391 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 10 Oct 2024 11:33:19 +0200 Subject: [PATCH 53/55] adapte logger warning val --- qgepqwat2ili/qgep/export.py | 4 +++- qgepqwat2ili/qgepdss/export.py | 1 + qgepqwat2ili/qgepsia405/export.py | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 6f9a74ad..0e3e1531 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -102,7 +102,9 @@ def truncate(val, max_length): if val is None: return None if len(val) > max_length: - logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + # _log() got an unexpected keyword argument 'stacklevel' + # logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + logger.warning(f"Value '{val}' exceeds expected length ({max_length})") return val[0:max_length] def modulo_angle(val): diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 67c53595..2b711c94 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -6,6 +6,7 @@ from sqlalchemy.sql import text from .. import utils + # 4.10.2024 from ..utils.ili2db import skip_wwtp_structure_ids from ..utils.various import logger diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index e64f7a85..acdbff18 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -61,6 +61,7 @@ def get_tid(relation): """ if relation is None: return None + return tid_maker.tid_for_row(relation) def get_vl(relation): @@ -102,7 +103,9 @@ def truncate(val, max_length): if val is None: return None if len(val) > max_length: - logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + # _log() got an unexpected keyword argument 'stacklevel' + # logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + logger.warning(f"Value '{val}' exceeds expected length ({max_length})") return val[0:max_length] def modulo_angle(val): From 9c2d50212c7ef621087541c578e069694f5d2084 Mon Sep 17 00:00:00 2001 From: Arnaud Poncet-Montanges Date: Thu, 10 Oct 2024 15:19:20 +0200 Subject: [PATCH 54/55] fix flake8 --- qgepqwat2ili/utils/ili2db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 36fe6fb8..04e2f3dd 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -469,7 +469,7 @@ def skip_wwtp_structure_ids(): # select all obj_id from wastewater_structure that are not in wwtp_structure cursor.execute( - f"SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" + "SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" ) # remove - only for testing # cursor.execute( From a29eb3ff3fa0b125681f45e2265b174d747259e6 Mon Sep 17 00:00:00 2001 From: SJiB Date: Thu, 10 Oct 2024 15:19:28 +0200 Subject: [PATCH 55/55] no f-string, as there are no placeholders --- qgepqwat2ili/utils/ili2db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index 36fe6fb8..04e2f3dd 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -469,7 +469,7 @@ def skip_wwtp_structure_ids(): # select all obj_id from wastewater_structure that are not in wwtp_structure cursor.execute( - f"SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" + "SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" ) # remove - only for testing # cursor.execute(