Skip to content

Commit

Permalink
Refactor reindex helpers in an importable way
Browse files Browse the repository at this point in the history
  • Loading branch information
reebalazs committed Sep 22, 2023
1 parent 4e97097 commit 8725fa8
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 72 deletions.
56 changes: 56 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,62 @@ Example value:

If needed, the default [`kitconcept.solr.interfaces.IKitconceptSolrSettings`](./src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml) can be customized in the registry via GenericSetup.

### Using reindex helpers

Helpers for activate and reindex solr are importable from the package.

Example for a reindex script that can be called from Makefile:

```py
from kitconcept.solr.reindex_helpers import activate_and_reindex
from Testing.makerequest import makerequest
from zope.site.hooks import setSite

import sys
import transaction


if __name__ == "__main__":
app = makerequest(app) # noQA

# Set site to Plone
site_id = "Plone"
portal = app.unrestrictedTraverse(site_id)
setSite(portal)

activate_and_reindex(portal, clear="--clear" in sys.argv)

transaction.commit()
app._p_jar.sync()
```

Example for an upgrade step that adds the `kitconcept.solr` package, and one that does the solr activation for the first time:

```py
from kitconcept.solr.reindex_helpers import activate_and_reindex
from plone import api

import logging


logger = logging.getLogger("your_package_name_here")


# We suggest to add two distinct upgrade step for the package installation
# and the solr activation, in case of a failure this allows to
# identify the problem easier.


def install_kitconcept_solr(context):
st = api.portal.get_tool("portal_setup")
st.runAllImportStepsFromProfile("kitconcept.solr:default")
logger.info("Installed kitconcept.solr")

def activate_and_reindex_solr(context):
activate_and_reindex(context)
logger.info("Acticated and reindexed solr")
```

### Update translations

```bash
Expand Down
82 changes: 10 additions & 72 deletions scripts/solr_activate_and_reindex.py
Original file line number Diff line number Diff line change
@@ -1,82 +1,20 @@
from collective.solr.interfaces import ISolrConnectionManager
from plone.registry.interfaces import IRegistry
from kitconcept.solr.reindex_helpers import activate_and_reindex
from Testing.makerequest import makerequest
from zope.component import getUtility
from zope.component import queryUtility
from zope.site.hooks import setSite

import logging
import sys
import transaction


logger = logging.getLogger("kitconcept.solr")
logger.setLevel(logging.DEBUG)
if __name__ == "__main__":
app = makerequest(app) # noQA

indexer_logger = logging.getLogger("collective.solr.indexer")
# Set site to Plone
site_id = "Plone"
portal = app.unrestrictedTraverse(site_id)
setSite(portal)

activate_and_reindex(portal, clear="--clear" in sys.argv)

def solr_is_running(portal):
manager = queryUtility(ISolrConnectionManager, context=portal)
schema = manager.getSchema()
return schema is not None


def solr_must_be_running(portal):
if not solr_is_running(portal):
logger.fatal("*** Solr must be running! (make solr-start) ***")
sys.exit(1)


def activate(active=True):
"""(de)activate the solr integration"""
registry = getUtility(IRegistry)
registry["collective.solr.active"] = active


def silence_logger():
orig_logger_exception = indexer_logger.exception

def new_logger_exception(msg):
if msg != "Error occured while getting data for indexing!":
orig_logger_exception(msg)

indexer_logger.exception = new_logger_exception

def reactivate_logger():
indexer_logger.exception = orig_logger_exception

return reactivate_logger


def reindex(portal):
"""reindex the existing content in solr"""
maintenance = portal.unrestrictedTraverse("@@solr-maintenance")
if "--clear" in sys.argv:
logger.info("Clearing solr...")
maintenance.clear()
# Avoid throwing a lot of errors which are actually not errors,
# but the indexer keeps throwing them when it tries to traverse everything.
reactivate_logger = silence_logger()
logger.info("Reindexing solr...")
maintenance.reindex()
reactivate_logger()


app = makerequest(app) # noQA

# Set site to Plone
site_id = "Plone"
portal = app.unrestrictedTraverse(site_id)
setSite(portal)

# Activate before confirming solr is running,
# because the confirmation only works if solr is enabled in the registry.
# If solr isn't running, we'll exit
# before committing the transaction with the activation.
activate()
solr_must_be_running(portal)
reindex(portal)

transaction.commit()
app._p_jar.sync()
transaction.commit()
app._p_jar.sync()
68 changes: 68 additions & 0 deletions src/kitconcept/solr/reindex_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
from collective.solr.interfaces import ISolrConnectionManager
from plone import api
from zope.component import queryUtility

import logging


logger = logging.getLogger("kitconcept.solr")
logger.setLevel(logging.DEBUG)

indexer_logger = logging.getLogger("collective.solr.indexer")


def solr_is_running(portal):
manager = queryUtility(ISolrConnectionManager, context=portal)
schema = manager.getSchema()
return schema is not None


def solr_must_be_running(portal):
if not solr_is_running(portal):
logger.fatal("*** Solr must be running! (make solr-start) ***")
return False
return True


def activate(active=True):
"""(de)activate the solr integration"""
api.portal.set_registry_record("collective.solr.active", active)


def silence_logger():
orig_logger_exception = indexer_logger.exception

def new_logger_exception(msg):
if msg != "Error occured while getting data for indexing!":
orig_logger_exception(msg)

indexer_logger.exception = new_logger_exception

def reactivate_logger():
indexer_logger.exception = orig_logger_exception

return reactivate_logger


def reindex(portal, clear=False):
"""reindex the existing content in solr"""
maintenance = portal.unrestrictedTraverse("@@solr-maintenance")
if clear:
logger.info("Clearing solr...")
maintenance.clear()
# Avoid throwing a lot of errors which are actually not errors,
# but the indexer keeps throwing them when it tries to traverse everything.
reactivate_logger = silence_logger()
logger.info("Reindexing solr...")
maintenance.reindex()
reactivate_logger()


def activate_and_reindex(portal, clear=False):
# Activate before confirming solr is running,
# because the confirmation only works if solr is enabled in the registry.
# If solr isn't running, we'll exit
# before committing the transaction with the activation.
activate()
if solr_must_be_running(portal):
reindex(portal, clear=clear)

0 comments on commit 8725fa8

Please sign in to comment.