From dca2f1b28cd7aa486357ef5f95c7399c7aea4562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Re=C3=A9?= Date: Fri, 22 Sep 2023 17:23:30 +0200 Subject: [PATCH] Refactor reindex helpers in an importable way --- README.md | 57 ++++++++++++++++++ scripts/solr_activate_and_reindex.py | 82 ++++---------------------- src/kitconcept/solr/reindex_helpers.py | 68 +++++++++++++++++++++ 3 files changed, 135 insertions(+), 72 deletions(-) create mode 100644 src/kitconcept/solr/reindex_helpers.py diff --git a/README.md b/README.md index 66bab87..ff273aa 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,63 @@ Example value: If needed, the default [`kitconcept.solr.interfaces.IKitconceptSolrSettings`](./src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml) can be customized in the registry via GenericSetup. +### Using reindex helpers + +Helpers for activate and reindex solr are importable from the package. + +Example for a reindex script that can be called from Makefile: + +```py +from kitconcept.solr.reindex_helpers import activate_and_reindex +from Testing.makerequest import makerequest +from zope.site.hooks import setSite + +import sys +import transaction + + +if __name__ == "__main__": + app = makerequest(app) # noQA + + # Set site to Plone + site_id = "Plone" + portal = app.unrestrictedTraverse(site_id) + setSite(portal) + + activate_and_reindex(portal, clear="--clear" in sys.argv) + + transaction.commit() + app._p_jar.sync() +``` + +Example for an upgrade step that adds the `kitconcept.solr` package, and one that does the solr activation for the first time: + +```py +from kitconcept.solr.reindex_helpers import activate_and_reindex +from plone import api + +import logging + + +logger = logging.getLogger("your_package_name_here") + + +# We suggest to add two distinct upgrade step for the package installation +# and the solr activation, in case of a failure this allows to +# identify the problem easier. + + +def install_kitconcept_solr(context): + st = api.portal.get_tool("portal_setup") + st.runAllImportStepsFromProfile("kitconcept.solr:default") + logger.info("Installed kitconcept.solr") + + +def activate_and_reindex_solr(context): + activate_and_reindex(context) + logger.info("Activated and reindexed solr") +``` + ### Update translations ```bash diff --git a/scripts/solr_activate_and_reindex.py b/scripts/solr_activate_and_reindex.py index 4ecf605..96ff3c2 100644 --- a/scripts/solr_activate_and_reindex.py +++ b/scripts/solr_activate_and_reindex.py @@ -1,82 +1,20 @@ -from collective.solr.interfaces import ISolrConnectionManager -from plone.registry.interfaces import IRegistry +from kitconcept.solr.reindex_helpers import activate_and_reindex from Testing.makerequest import makerequest -from zope.component import getUtility -from zope.component import queryUtility from zope.site.hooks import setSite -import logging import sys import transaction -logger = logging.getLogger("kitconcept.solr") -logger.setLevel(logging.DEBUG) +if __name__ == "__main__": + app = makerequest(app) # noQA -indexer_logger = logging.getLogger("collective.solr.indexer") + # Set site to Plone + site_id = "Plone" + portal = app.unrestrictedTraverse(site_id) + setSite(portal) + activate_and_reindex(portal, clear="--clear" in sys.argv) -def solr_is_running(portal): - manager = queryUtility(ISolrConnectionManager, context=portal) - schema = manager.getSchema() - return schema is not None - - -def solr_must_be_running(portal): - if not solr_is_running(portal): - logger.fatal("*** Solr must be running! (make solr-start) ***") - sys.exit(1) - - -def activate(active=True): - """(de)activate the solr integration""" - registry = getUtility(IRegistry) - registry["collective.solr.active"] = active - - -def silence_logger(): - orig_logger_exception = indexer_logger.exception - - def new_logger_exception(msg): - if msg != "Error occured while getting data for indexing!": - orig_logger_exception(msg) - - indexer_logger.exception = new_logger_exception - - def reactivate_logger(): - indexer_logger.exception = orig_logger_exception - - return reactivate_logger - - -def reindex(portal): - """reindex the existing content in solr""" - maintenance = portal.unrestrictedTraverse("@@solr-maintenance") - if "--clear" in sys.argv: - logger.info("Clearing solr...") - maintenance.clear() - # Avoid throwing a lot of errors which are actually not errors, - # but the indexer keeps throwing them when it tries to traverse everything. - reactivate_logger = silence_logger() - logger.info("Reindexing solr...") - maintenance.reindex() - reactivate_logger() - - -app = makerequest(app) # noQA - -# Set site to Plone -site_id = "Plone" -portal = app.unrestrictedTraverse(site_id) -setSite(portal) - -# Activate before confirming solr is running, -# because the confirmation only works if solr is enabled in the registry. -# If solr isn't running, we'll exit -# before committing the transaction with the activation. -activate() -solr_must_be_running(portal) -reindex(portal) - -transaction.commit() -app._p_jar.sync() + transaction.commit() + app._p_jar.sync() diff --git a/src/kitconcept/solr/reindex_helpers.py b/src/kitconcept/solr/reindex_helpers.py new file mode 100644 index 0000000..c311274 --- /dev/null +++ b/src/kitconcept/solr/reindex_helpers.py @@ -0,0 +1,68 @@ +from collective.solr.interfaces import ISolrConnectionManager +from plone import api +from zope.component import queryUtility + +import logging + + +logger = logging.getLogger("kitconcept.solr") +logger.setLevel(logging.DEBUG) + +indexer_logger = logging.getLogger("collective.solr.indexer") + + +def solr_is_running(portal): + manager = queryUtility(ISolrConnectionManager, context=portal) + schema = manager.getSchema() + return schema is not None + + +def solr_must_be_running(portal): + if not solr_is_running(portal): + logger.fatal("*** Solr must be running! (make solr-start) ***") + return False + return True + + +def activate(active=True): + """(de)activate the solr integration""" + api.portal.set_registry_record("collective.solr.active", active) + + +def silence_logger(): + orig_logger_exception = indexer_logger.exception + + def new_logger_exception(msg): + if msg != "Error occured while getting data for indexing!": + orig_logger_exception(msg) + + indexer_logger.exception = new_logger_exception + + def reactivate_logger(): + indexer_logger.exception = orig_logger_exception + + return reactivate_logger + + +def reindex(portal, clear=False): + """reindex the existing content in solr""" + maintenance = portal.unrestrictedTraverse("@@solr-maintenance") + if clear: + logger.info("Clearing solr...") + maintenance.clear() + # Avoid throwing a lot of errors which are actually not errors, + # but the indexer keeps throwing them when it tries to traverse everything. + reactivate_logger = silence_logger() + logger.info("Reindexing solr...") + maintenance.reindex() + reactivate_logger() + + +def activate_and_reindex(portal, clear=False): + # Activate before confirming solr is running, + # because the confirmation only works if solr is enabled in the registry. + # If solr isn't running, we'll exit + # before committing the transaction with the activation. + activate() + if solr_must_be_running(portal): + reindex(portal, clear=clear)