From 4829ffc88f0070bbf1336ea99556405f7b75c01c Mon Sep 17 00:00:00 2001 From: Hiroyasu OHYAMA Date: Mon, 9 Dec 2024 08:01:08 +0000 Subject: [PATCH 1/3] Fixed problem that might raise ElasticsearchException depends on context by SearchChain processing --- CHANGELOG.md | 3 +++ api_v1/entry/serializer.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92ee21c94..dc6672311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ ### Changed ### Fixed +* Fixed problem that might raise ElasticsearchException depends on context by + SearchChain processing. + Contributed by @hinashi, @userlocalhost ## v3.110.0 diff --git a/api_v1/entry/serializer.py b/api_v1/entry/serializer.py index ed80b71ee..0e3baf2e9 100644 --- a/api_v1/entry/serializer.py +++ b/api_v1/entry/serializer.py @@ -302,14 +302,21 @@ def _do_forward_search(sub_query, sub_query_result): ] # get Entry informations from result + # NOTE: small limit(=1000) is workaround to prevent overload try: search_result = AdvancedSearchService.search_entries( - user, entity_id_list, hint_attrs, limit=99999 + user, + entity_id_list, + hint_attrs, + limit=CONFIG.SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT, ) except Exception as e: Logger.warning("Search Chain API error:%s" % e) raise ElasticsearchException() + # All results of this request would be joined and pass to next request. It might be + # huge request and leads to glitch of Elasticsearch by just a single request. + # This is our original curcit breaker to prevent overload because of them. if search_result.ret_count > CONFIG.SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT: Logger.warning("Search Chain API error: SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT") raise ElasticsearchException() From eb6066ec4b278787ca5eee5b2cac0424dbc76915 Mon Sep 17 00:00:00 2001 From: Hiroyasu OHYAMA Date: Mon, 9 Dec 2024 08:39:13 +0000 Subject: [PATCH 2/3] Fixed to get actual retrieved count to compare with specified threshold. The .ret_count of search_result contains total number of items that is associated with specified request in the Elasticsearch. And the len(.ret_values) represents number of items that are actually retrieved. --- api_v1/entry/serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_v1/entry/serializer.py b/api_v1/entry/serializer.py index 0e3baf2e9..a2eba56cc 100644 --- a/api_v1/entry/serializer.py +++ b/api_v1/entry/serializer.py @@ -317,7 +317,7 @@ def _do_forward_search(sub_query, sub_query_result): # All results of this request would be joined and pass to next request. It might be # huge request and leads to glitch of Elasticsearch by just a single request. # This is our original curcit breaker to prevent overload because of them. - if search_result.ret_count > CONFIG.SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT: + if len(search_result.ret_values) > CONFIG.SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT: Logger.warning("Search Chain API error: SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT") raise ElasticsearchException() From 92e8ca39c1027f788eb63eedaaf00e3ce014a907 Mon Sep 17 00:00:00 2001 From: Hiroyasu OHYAMA Date: Mon, 9 Dec 2024 09:01:44 +0000 Subject: [PATCH 3/3] Skip unnecessary test because of changing implementation --- api_v1/tests/entry/test_api_search_chain.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api_v1/tests/entry/test_api_search_chain.py b/api_v1/tests/entry/test_api_search_chain.py index 44cc683c7..78f2faab9 100644 --- a/api_v1/tests/entry/test_api_search_chain.py +++ b/api_v1/tests/entry/test_api_search_chain.py @@ -1,6 +1,6 @@ import copy import json -from unittest import mock +from unittest import mock, skip from airone.lib.test import AironeViewTest from airone.lib.types import AttrType @@ -1326,6 +1326,10 @@ def test_search_backward_chain_exceeding_search_limit(self): ), ) + @skip(""" + A situation that raises ElasticsearchException because of exceeding search count because of + installing workaround limitation cap won't be happened. + """) def test_search_chain_when_result_exceeds_acceptable_count(self): # Change configuration to test processing for acceptable result from elasticsearch ENTRY_CONFIG.conf["SEARCH_CHAIN_ACCEPTABLE_RESULT_COUNT"] = 2