From 763e51c4f193ddb7a1449d531b884ccc72c851e2 Mon Sep 17 00:00:00 2001 From: Sawy Date: Sun, 8 Sep 2024 10:54:16 +0300 Subject: [PATCH 1/4] CSV-5225: ignore nested documents that exceeds nested_objects_limit --- app/lib/check_elastic_search.rb | 1 + app/models/concerns/annotation_base.rb | 7 +++++++ app/models/tipline_request.rb | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/app/lib/check_elastic_search.rb b/app/lib/check_elastic_search.rb index 00cb5681ab..35268006bb 100644 --- a/app/lib/check_elastic_search.rb +++ b/app/lib/check_elastic_search.rb @@ -82,6 +82,7 @@ def get_fresh_value(data) def add_update_nested_obj(options) return if self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] + return if options[:op] == 'create' && self.respond_to?(:hit_nested_objects_limit?) && self.hit_nested_objects_limit? model = { klass: self.class.name, id: self.id } ElasticSearchWorker.perform_in(1.second, YAML::dump(model), YAML::dump(options), 'create_update_doc_nested') end diff --git a/app/models/concerns/annotation_base.rb b/app/models/concerns/annotation_base.rb index 20636fbc0d..ad3cebb944 100644 --- a/app/models/concerns/annotation_base.rb +++ b/app/models/concerns/annotation_base.rb @@ -156,6 +156,13 @@ def custom_permissions(ability = nil) perms["destroy Smooch"] = ability.can?(:destroy, self) if self.annotation_type == 'smooch' perms end + + def hit_nested_objects_limit? + ret = false + pm = self.project_media + ret = pm.get_annotations(self.annotation_type).count > CheckConfig.get('nested_objects_limit', 10000, :integer) unless pm.nil? + ret + end end module ClassMethods diff --git a/app/models/tipline_request.rb b/app/models/tipline_request.rb index 385f64d258..ad0fb75147 100644 --- a/app/models/tipline_request.rb +++ b/app/models/tipline_request.rb @@ -64,6 +64,11 @@ def associated_graphql_id Base64.encode64("#{self.associated_type}/#{self.associated_id}") end + def hit_nested_objects_limit? + associated = self.associated + associated.tipline_requests.count > CheckConfig.get('nested_objects_limit', 10000, :integer) + end + private def set_team_and_user From ad95f02b1654cdaa745e6c50618455c6108d2156 Mon Sep 17 00:00:00 2001 From: Sawy Date: Sun, 8 Sep 2024 21:59:45 +0300 Subject: [PATCH 2/4] CV2-5225: add tests --- test/controllers/elastic_search_7_test.rb | 24 +++++++++++++++++++ test/controllers/elastic_search_9_test.rb | 29 ++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/test/controllers/elastic_search_7_test.rb b/test/controllers/elastic_search_7_test.rb index dca282fad6..236c33c2bf 100644 --- a/test/controllers/elastic_search_7_test.rb +++ b/test/controllers/elastic_search_7_test.rb @@ -300,5 +300,29 @@ def setup end end + test "should ignore index task responses that exceeds nested objects limit" do + team = create_team + tt = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_a', 'ans_b', 'ans_c'] + tt2 = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_aa', 'ans_bb', 'ans_cc'] + tt3 = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_aaa', 'ans_bbb', 'ans_ccc'] + pm = create_project_media team: team + stub_configs({ 'nested_objects_limit' => 2 }) do + # answer single choice + puts "Start to answer" + pm_tt = pm.annotations('task').select{|t| t.team_task_id == tt.id}.last + pm_tt.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_a' }.to_json }.to_json + pm_tt.save! + # pm_tt2 = pm.annotations('task').select{|t| t.team_task_id == tt2.id}.last + # pm_tt2.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_aa' }.to_json }.to_json + # pm_tt2.save! + # pm_tt3 = pm.annotations('task').select{|t| t.team_task_id == tt3.id}.last + # pm_tt3.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_aaa' }.to_json }.to_json + # pm_tt3.save! + sleep 2 + es = $repository.find(pm.get_es_doc_id) + pp es['task_responses'] + end + end + # Please add new tests to test/controllers/elastic_search_8_test.rb end diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index a015628a52..3f14d8f219 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -206,7 +206,7 @@ def setup test "shoud add team filter by default" do t = create_team t2 = create_team - pm1 = create_project_media team: t, quote: 'test', disable_es_callbacks: false + pm1 = create_project_media team: t, quote: 'test', disable_es_callbacks: false pm2 = create_project_media team: t2, quote: 'test', disable_es_callbacks: false ProjectMedia.where(id: [pm1.id, pm2.id]).update_all(project_id: nil) options = { @@ -230,5 +230,32 @@ def setup Team.unstub(:current) end + test "should ignore index document that exceeds nested objects limit" do + team = create_team + pm = create_project_media team: team + stub_configs({ 'nested_objects_limit' => 2 }) do + tr = create_tipline_request associated: pm, disable_es_callbacks: false + tr2 = create_tipline_request associated: pm, disable_es_callbacks: false + tr3 = create_tipline_request associated: pm, disable_es_callbacks: false + t = create_tag annotated: pm, disable_es_callbacks: false + t2 = create_tag annotated: pm, disable_es_callbacks: false + t3 = create_tag annotated: pm, disable_es_callbacks: false + c = create_comment annotated: pm, disable_es_callbacks: false + c2 = create_comment annotated: pm, disable_es_callbacks: false + c3 = create_comment annotated: pm, disable_es_callbacks: false + sleep 2 + es = $repository.find(pm.get_es_doc_id) + requests = es['requests'] + assert_equal 2, requests.size + assert_equal [tr.id, tr2.id], requests.collect{|r| r['id']}.sort + tags = es['tags'] + assert_equal 2, tags.size + assert_equal [t.id, t2.id], tags.collect{|i| i['id']}.sort + comments = es['comments'] + assert_equal 2, comments.size + assert_equal [c.id, c2.id], comments.collect{|i| i['id']}.sort + end + end + # Please add new tests to test/controllers/elastic_search_10_test.rb end From 5c74496a3e7ca5d67f0fa37855a08a0e43ebc2e0 Mon Sep 17 00:00:00 2001 From: Sawy Date: Sun, 8 Sep 2024 22:26:41 +0300 Subject: [PATCH 3/4] CV2-5225: cleanup --- test/controllers/elastic_search_7_test.rb | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/controllers/elastic_search_7_test.rb b/test/controllers/elastic_search_7_test.rb index 236c33c2bf..a10d4dee85 100644 --- a/test/controllers/elastic_search_7_test.rb +++ b/test/controllers/elastic_search_7_test.rb @@ -302,25 +302,25 @@ def setup test "should ignore index task responses that exceeds nested objects limit" do team = create_team - tt = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_a', 'ans_b', 'ans_c'] - tt2 = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_aa', 'ans_bb', 'ans_cc'] - tt3 = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_aaa', 'ans_bbb', 'ans_ccc'] - pm = create_project_media team: team stub_configs({ 'nested_objects_limit' => 2 }) do - # answer single choice - puts "Start to answer" + tt = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_a', 'ans_b', 'ans_c'] + tt2 = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_aa', 'ans_bb', 'ans_cc'] + tt3 = create_team_task team_id: team.id, type: 'free_text' + pm = create_project_media team: team, disable_es_callbacks: false pm_tt = pm.annotations('task').select{|t| t.team_task_id == tt.id}.last pm_tt.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_a' }.to_json }.to_json pm_tt.save! - # pm_tt2 = pm.annotations('task').select{|t| t.team_task_id == tt2.id}.last - # pm_tt2.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_aa' }.to_json }.to_json - # pm_tt2.save! - # pm_tt3 = pm.annotations('task').select{|t| t.team_task_id == tt3.id}.last - # pm_tt3.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_aaa' }.to_json }.to_json - # pm_tt3.save! + pm_tt2 = pm.annotations('task').select{|t| t.team_task_id == tt2.id}.last + pm_tt2.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_aa' }.to_json }.to_json + pm_tt2.save! + pm_tt3 = pm.annotations('task').select{|t| t.team_task_id == tt3.id}.last + pm_tt3.response = { annotation_type: 'task_response_free_text', set_fields: { response_free_text: 'Foo by Sawy' }.to_json }.to_json + pm_tt3.save! sleep 2 es = $repository.find(pm.get_es_doc_id) - pp es['task_responses'] + task_responses = es['task_responses'] + # TODO: fix the test + # assert_equal 2, task_responses.size end end From c5330248637d5e771a61875a0be8d2527a86b0c8 Mon Sep 17 00:00:00 2001 From: Sawy Date: Mon, 9 Sep 2024 07:02:52 +0300 Subject: [PATCH 4/4] CV2-5225: apply nested_objects_limit on tags and requests --- app/models/annotations/tag.rb | 7 +++++++ app/models/concerns/annotation_base.rb | 7 ------- test/controllers/elastic_search_7_test.rb | 24 ----------------------- test/controllers/elastic_search_9_test.rb | 6 ------ 4 files changed, 7 insertions(+), 37 deletions(-) diff --git a/app/models/annotations/tag.rb b/app/models/annotations/tag.rb index d365976f03..4755b39d62 100644 --- a/app/models/annotations/tag.rb +++ b/app/models/annotations/tag.rb @@ -84,6 +84,13 @@ def self.run_bulk_create_callbacks(ids_json, pmids_json) ProjectMedia.where(id: pmids).find_each { |pm| pm.tags_as_sentence } end + def hit_nested_objects_limit? + ret = false + pm = self.project_media + ret = pm.get_annotations(self.annotation_type).count > CheckConfig.get('nested_objects_limit', 10000, :integer) unless pm.nil? + ret + end + private def get_tag_text_reference diff --git a/app/models/concerns/annotation_base.rb b/app/models/concerns/annotation_base.rb index ad3cebb944..20636fbc0d 100644 --- a/app/models/concerns/annotation_base.rb +++ b/app/models/concerns/annotation_base.rb @@ -156,13 +156,6 @@ def custom_permissions(ability = nil) perms["destroy Smooch"] = ability.can?(:destroy, self) if self.annotation_type == 'smooch' perms end - - def hit_nested_objects_limit? - ret = false - pm = self.project_media - ret = pm.get_annotations(self.annotation_type).count > CheckConfig.get('nested_objects_limit', 10000, :integer) unless pm.nil? - ret - end end module ClassMethods diff --git a/test/controllers/elastic_search_7_test.rb b/test/controllers/elastic_search_7_test.rb index a10d4dee85..dca282fad6 100644 --- a/test/controllers/elastic_search_7_test.rb +++ b/test/controllers/elastic_search_7_test.rb @@ -300,29 +300,5 @@ def setup end end - test "should ignore index task responses that exceeds nested objects limit" do - team = create_team - stub_configs({ 'nested_objects_limit' => 2 }) do - tt = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_a', 'ans_b', 'ans_c'] - tt2 = create_team_task team_id: team.id, type: 'single_choice', options: ['ans_aa', 'ans_bb', 'ans_cc'] - tt3 = create_team_task team_id: team.id, type: 'free_text' - pm = create_project_media team: team, disable_es_callbacks: false - pm_tt = pm.annotations('task').select{|t| t.team_task_id == tt.id}.last - pm_tt.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_a' }.to_json }.to_json - pm_tt.save! - pm_tt2 = pm.annotations('task').select{|t| t.team_task_id == tt2.id}.last - pm_tt2.response = { annotation_type: 'task_response_single_choice', set_fields: { response_single_choice: 'ans_aa' }.to_json }.to_json - pm_tt2.save! - pm_tt3 = pm.annotations('task').select{|t| t.team_task_id == tt3.id}.last - pm_tt3.response = { annotation_type: 'task_response_free_text', set_fields: { response_free_text: 'Foo by Sawy' }.to_json }.to_json - pm_tt3.save! - sleep 2 - es = $repository.find(pm.get_es_doc_id) - task_responses = es['task_responses'] - # TODO: fix the test - # assert_equal 2, task_responses.size - end - end - # Please add new tests to test/controllers/elastic_search_8_test.rb end diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index 3f14d8f219..6493472a94 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -240,9 +240,6 @@ def setup t = create_tag annotated: pm, disable_es_callbacks: false t2 = create_tag annotated: pm, disable_es_callbacks: false t3 = create_tag annotated: pm, disable_es_callbacks: false - c = create_comment annotated: pm, disable_es_callbacks: false - c2 = create_comment annotated: pm, disable_es_callbacks: false - c3 = create_comment annotated: pm, disable_es_callbacks: false sleep 2 es = $repository.find(pm.get_es_doc_id) requests = es['requests'] @@ -251,9 +248,6 @@ def setup tags = es['tags'] assert_equal 2, tags.size assert_equal [t.id, t2.id], tags.collect{|i| i['id']}.sort - comments = es['comments'] - assert_equal 2, comments.size - assert_equal [c.id, c2.id], comments.collect{|i| i['id']}.sort end end