Further improvement of Performance (#1544)

* load glossaries, municipalities and disclaimers on server start to reduce db queries

* fix lint, comment profiler

* fix lint

dev-requirements.txt

@@ -6,4 +6,5 @@ McCabe==0.6.1
 Sphinx<1.6
 sphinx_rtd_theme
 c2c.template==2.3.0
+yappi
 -e .

docker-compose.yml

@@ -43,6 +43,7 @@ services:
       - .:/workspace:cached
       - venvs:/venvs
     working_dir: /workspace
+    command: [ "/venvs/.venv/bin/pserve", "development.ini", "--reload"]
     ports:
       - ${PYRAMID_OEREB_PORT:-6543}:${PYRAMID_OEREB_PORT:-6543}
     networks:

pyramid_oereb/contrib/data_sources/standard/sources/disclaimer.py

@@ -5,14 +5,11 @@
 
 class DatabaseSource(BaseDatabaseSource, DisclaimerBaseSource):
 
-    def read(self, params):
+    def read(self):
         """
         The read method to access the standard database structure. It uses SQL-Alchemy for querying. It does
         not accept any parameters nor it applies any filter on the database query. It simply loads all
         content from the configured model.
-
-        Args:
-            params (pyramid_oereb.views.webservice.Parameter): The parameters of the extract request.
         """
         session = self._adapter_.get_session(self._key_)
         try:

pyramid_oereb/contrib/data_sources/standard/sources/glossary.py

@@ -5,12 +5,9 @@
 
 class DatabaseSource(BaseDatabaseSource, GlossaryBaseSource):
 
-    def read(self, params):
+    def read(self):
         """
         Central method to read all glossary entries.
-
-        Args:
-            params (pyramid_oereb.views.webservice.Parameter): The parameters of the extract request.
         """
         session = self._adapter_.get_session(self._key_)
         try:

pyramid_oereb/contrib/data_sources/standard/sources/municipality.py

@@ -10,21 +10,16 @@
 
 class DatabaseSource(BaseDatabaseSource, MunicipalityBaseSource):
 
-    def read(self, params, fosnr=None):
+    def read(self):
         """
-        Central method to read a municipality by it's id_bfs identifier.
-
-        Args:
-            params (pyramid_oereb.views.webservice.Parameter): The parameters of the extract request.
-            fosnr (int or None): The federal number of the municipality defined by the statistics office.
+        The read method to access the standard database structure. It uses SQL-Alchemy for querying. It does
+        not accept any parameters nor it applies any filter on the database query. It simply loads all
+        content from the configured model.
         """
         session = self._adapter_.get_session(self._key_)
         try:
             self.records = list()
-            if fosnr:
-                results = session.query(self._model_).filter(self._model_.fosnr == fosnr).all()
-            else:
-                results = session.query(self._model_).all()
+            results = session.query(self._model_).all()
             for result in results:
                 self.records.append(self._record_class_(
                     result.fosnr,

pyramid_oereb/core/config.py

@@ -6,6 +6,9 @@
 from pyramid.config import ConfigurationError
 
 from pyramid_oereb.core.readers.availability import AvailabilityReader
+from pyramid_oereb.core.readers.disclaimer import DisclaimerReader
+from pyramid_oereb.core.readers.glossary import GlossaryReader
+from pyramid_oereb.core.readers.municipality import MunicipalityReader
 from pyramid_oereb.core.records.office import OfficeRecord
 from pyramid_oereb.core.records.document_types import DocumentTypeRecord
 from pyramid_oereb.core.records.law_status import LawStatusRecord
@@ -45,6 +48,9 @@ class Config(object):
     theme_document = None
     offices = None
     availabilities = None
+    glossaries = None
+    disclaimers = None
+    municipalities = None
 
     @staticmethod
     def init(configfile, configsection, c2ctemplate_style=False, init_data=False):
@@ -75,6 +81,9 @@ def init(configfile, configsection, c2ctemplate_style=False, init_data=False):
             Config.init_logos()
             Config.assemble_relation_themes_documents()
             Config.init_availabilities()
+            Config.init_glossaries()
+            Config.init_disclaimers()
+            Config.init_municipalities()
 
     @staticmethod
     def get_config():
@@ -183,6 +192,18 @@ def init_availabilities():
         """
         Config.availabilities = Config._read_availabilities()
 
+    @staticmethod
+    def init_glossaries():
+        Config.glossaries = Config._read_glossaries()
+
+    @staticmethod
+    def init_disclaimers():
+        Config.disclaimers = Config._read_disclaimers()
+
+    @staticmethod
+    def init_municipalities():
+        Config.municipalities = Config._read_municipalities()
+
     @staticmethod
     def assemble_relation_themes_documents():
         """
@@ -365,6 +386,72 @@ def _read_availabilities():
         )
         return availability_reader.read()
 
+    @staticmethod
+    def _read_glossaries():
+        """
+        Reads settings of glossaries from configured source and instantiates the relevant reader.
+
+        Returns:
+            list of pyramid_oereb.core.records.glossary.GlossaryRecord:
+                The list of found records. Since these are not filtered by any criteria the list simply
+                contains all records delivered by the source.
+
+        Raises:
+            ConfigurationError
+        """
+        glossary_config = Config.get_glossary_config()
+        if glossary_config is None:
+            raise ConfigurationError("Missing configuration for glossary source config")
+        glossary_reader = GlossaryReader(
+            glossary_config.get('source').get('class'),
+            **glossary_config.get('source').get('params')
+        )
+        return glossary_reader.read()
+
+    @staticmethod
+    def _read_disclaimers():
+        """
+        Reads settings of disclaimers from configured source and instantiates the relevant reader.
+
+        Returns:
+            list of pyramid_oereb.core.records.disclaimer.DisclaimerRecord:
+                The list of found records. Since these are not filtered by any criteria the list simply
+                contains all records delivered by the source.
+
+        Raises:
+            ConfigurationError
+        """
+        disclaimer_config = Config.get_disclaimer_config()
+        if disclaimer_config is None:
+            raise ConfigurationError("Missing configuration for disclaimer source config")
+        disclaimer_reader = DisclaimerReader(
+            disclaimer_config.get('source').get('class'),
+            **disclaimer_config.get('source').get('params')
+        )
+        return disclaimer_reader.read()
+
+    @staticmethod
+    def _read_municipalities():
+        """
+        Reads settings of disclaimers from configured source and instantiates the relevant reader.
+
+        Returns:
+            list of pyramid_oereb.core.records.municipality.MunicipalityRecord:
+                The list of found records. Since these are not filtered by any criteria the list simply
+                contains all records delivered by the source.
+
+        Raises:
+            ConfigurationError
+        """
+        municipality_config = Config.get_municipality_config()
+        if municipality_config is None:
+            raise ConfigurationError("Missing configuration for municipality source config")
+        municipality_reader = MunicipalityReader(
+            municipality_config.get('source').get('class'),
+            **municipality_config.get('source').get('params')
+        )
+        return municipality_reader.read()
+
     @staticmethod
     def get_srid():
         """
@@ -1948,6 +2035,31 @@ def availability_by_theme_code_municipality_fosnr(theme_code, fosnr):
                 return availability.available
         return True
 
+    @staticmethod
+    def municipality_by_fosnr(fosnr):
+        """
+        Loops through all configured municipalities read from configured source to find a match by
+        fosnr identifier.
+
+        Args:
+            fosnr (int): The key/fosnr which is used to find matching municipality.
+
+        Returns:
+            pyramid_oereb.lib.records.municipality.MunicipalityRecord: The found municipality
+        Raises:
+            ConfigurationError: If no match was found
+
+        """
+        for municipality in Config.municipalities:
+            if municipality.fosnr == fosnr:
+                return municipality
+        raise ConfigurationError(
+            'No municipalitiy with fosnr {} could be found in the configured municipalities ({}).'.format(
+                fosnr,
+                Config.municipalities
+            )
+        )
+
 
 def _parse(cfg_file, cfg_section, c2ctemplate_style=False):
     """

pyramid_oereb/core/processor.py

@@ -7,10 +7,7 @@
 
 from pyramid_oereb.core.config import Config
 from pyramid_oereb.core.records.plr import PlrRecord
-from pyramid_oereb.core.readers.disclaimer import DisclaimerReader
 from pyramid_oereb.core.readers.extract import ExtractReader
-from pyramid_oereb.core.readers.glossary import GlossaryReader
-from pyramid_oereb.core.readers.municipality import MunicipalityReader
 from pyramid_oereb.core.readers.real_estate import RealEstateReader
 
 
@@ -19,8 +16,7 @@
 
 class Processor(object):
 
-    def __init__(self, real_estate_reader, municipality_reader, disclaimer_reader,
-                 glossary_reader, plr_sources, extract_reader):
+    def __init__(self, real_estate_reader, plr_sources, extract_reader):
         """
         The Processor class is directly bound to the get_extract_by_id service in this application. It's task
         is to unsnarl the difficult model of the oereb extract and handle all objects inside this extract
@@ -30,22 +26,12 @@ def __init__(self, real_estate_reader, municipality_reader, disclaimer_reader,
         Args:
             real_estate_reader (pyramid_oereb.lib.readers.real_estate.RealEstateReader): The
                 real estate reader instance for runtime use.
-            municipality_reader (pyramid_oereb.lib.readers.municipality.MunicipalityReader): The
-                municipality reader instance for runtime use.
-            disclaimer_reader
-                (pyramid_oereb.core.readers.disclaimer.DisclaimerReader):
-                The disclaimer of liability reader instance for runtime use.
-            glossary_reader (pyramid_oereb.lib.readers.glossary.GlossaryReader): The glossary
-                reader instance for runtime use.
             plr_sources (list of pyramid_oereb.standard.sources.plr.DatabaseSource): The
                 public law restriction source instances for runtime use wrapped in a list.
             extract_reader (pyramid_oereb.lib.readers.extract.ExtractReader): The extract reader
                 instance for runtime use.
         """
         self._real_estate_reader_ = real_estate_reader
-        self._municipality_reader_ = municipality_reader
-        self._disclaimer_reader_ = disclaimer_reader
-        self._glossary_reader_ = glossary_reader
         self._plr_sources_ = plr_sources
         self._extract_reader_ = extract_reader
 
@@ -231,33 +217,6 @@ def real_estate_reader(self):
         """
         return self._real_estate_reader_
 
-    @property
-    def municipality_reader(self):
-        """
-        Returns:
-            pyramid_oereb.lib.readers.municipality.MunicipalityReader: The municipality reader
-            reader instance.
-        """
-        return self._municipality_reader_
-
-    @property
-    def disclaimer_reader(self):
-        """
-        Returns:
-            pyramid_oereb.lib.readers.disclaimer.DisclaimerReader:
-            The disclaimer reader reader instance.
-        """
-        return self._disclaimer_reader_
-
-    @property
-    def glossary_reader(self):
-        """
-        Returns:
-            pyramid_oereb.lib.readers.glossary.GlossaryReader: The glossary reader reader
-            instance.
-        """
-        return self._glossary_reader_
-
     @property
     def plr_sources(self):
         """
@@ -291,9 +250,7 @@ def process(self, real_estate, params, sld_url):
             pyramid_oereb.lib.records.extract.ExtractRecord: The generated extract record.
         """
         log.debug("process() start")
-        municipality = self._municipality_reader_.read(params, real_estate.fosnr)[0]
-        disclaimers = self._disclaimer_reader_.read(params)
-        glossaries = self._glossary_reader_.read(params)
+        municipality = Config.municipality_by_fosnr(real_estate.fosnr)
         extract_raw = self._extract_reader_.read(params, real_estate, municipality)
         extract = self.plr_tolerance_check(extract_raw)
 
@@ -312,8 +269,8 @@ def process(self, real_estate, params, sld_url):
         # intersecting and not intersecting and starts the legend entry sorting after.
         self.view_service_handling(extract.real_estate, params.images, params.format, params.language)
 
-        extract.disclaimers = disclaimers
-        extract.glossaries = glossaries
+        extract.disclaimers = Config.disclaimers
+        extract.glossaries = Config.glossaries
         log.debug("process() done, returning extract.")
         return extract
 
@@ -329,9 +286,6 @@ def create_processor():
     """
 
     real_estate_config = Config.get_real_estate_config()
-    municipality_config = Config.get_municipality_config()
-    disclaimer_config = Config.get_disclaimer_config()
-    glossary_config = Config.get_glossary_config()
 
     plr_cadastre_authority = Config.get_plr_cadastre_authority()
 
@@ -340,21 +294,6 @@ def create_processor():
         **real_estate_config.get('source').get('params')
     )
 
-    municipality_reader = MunicipalityReader(
-        municipality_config.get('source').get('class'),
-        **municipality_config.get('source').get('params')
-    )
-
-    disclaimer_reader = DisclaimerReader(
-        disclaimer_config.get('source').get('class'),
-        **disclaimer_config.get('source').get('params')
-    )
-
-    glossary_reader = GlossaryReader(
-        glossary_config.get('source').get('class'),
-        **glossary_config.get('source').get('params')
-    )
-
     plr_sources = []
     for plr in Config.get('plrs'):
         plr_source_class = DottedNameResolver().maybe_resolve(plr.get('source').get('class'))
@@ -367,9 +306,6 @@ def create_processor():
 
     return Processor(
         real_estate_reader=real_estate_reader,
-        municipality_reader=municipality_reader,
-        disclaimer_reader=disclaimer_reader,
-        glossary_reader=glossary_reader,
         plr_sources=plr_sources,
         extract_reader=extract_reader,
     )

pyramid_oereb/core/readers/disclaimer.py

@@ -24,7 +24,7 @@ class must exist and it must implement basic source behaviour of the
         source_class = DottedNameResolver().maybe_resolve(dotted_source_class_path)
         self._source_ = source_class(**params)
 
-    def read(self, params):
+    def read(self):
         """
         The read method of this reader. There we invoke the read method of the bound source.
 
@@ -34,13 +34,10 @@ def read(self, params):
             :ref:`api-pyramid_oereb-core-records-availability-availabilityrecord`. Otherwise the API like way
             the server works would be broken.
 
-        Args:
-            params (pyramid_oereb.views.webservice.Parameter): The parameters of the extract request.
-
         Returns:
             list of pyramid_oereb.core.records.disclaimer.DisclaimerRecord:
                 The list of found records. Since these are not filtered by any criteria the list simply
                 contains all records delivered by the source.
         """
-        self._source_.read(params)
+        self._source_.read()
         return self._source_.records