From 1598516ddac4b930a59f9283de75b510c043d5a0 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 28 Aug 2024 05:57:53 -0300 Subject: [PATCH 1/5] Search by multiple keywords regardless the order. Fixes: CV2-5129. --- app/models/team.rb | 6 +++++- test/models/team_2_test.rb | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/app/models/team.rb b/app/models/team.rb index 6ba123bb6b..13695de9db 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -535,7 +535,11 @@ def filtered_fact_checks(filters = {}) query = query.where('fact_checks.report_status' => filters[:report_status].to_a.map(&:to_s)) unless filters[:report_status].blank? # Filter by text - query = query.where('(fact_checks.title ILIKE ? OR fact_checks.url ILIKE ? OR fact_checks.summary ILIKE ?)', *["%#{filters[:text]}%"]*3) if filters[:text].to_s.size > 2 + if filters[:text].to_s.size > 2 + tsquery = Team.sanitize_sql_array(["to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages + tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || coalesce(url, ''))" + query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) + end # Exclude the ones already applied to a target item target = ProjectMedia.find_by_id(filters[:target_id].to_i) diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index 53275867ab..f9bceb24bc 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -1527,4 +1527,17 @@ def setup tbi.save! assert_equal ['none', 'link_preview'], t.available_newsletter_header_types end + + test "should search for fact-checks by keywords" do + Sidekiq::Testing.fake! + t = create_team + create_fact_check title: 'Some Other Test', claim_description: create_claim_description(project_media: create_project_media(team: t)) + create_fact_check title: 'Bar Bravo Foo Test', claim_description: create_claim_description(project_media: create_project_media(team: t)) + create_fact_check title: 'Foo Alpha Bar Test', claim_description: create_claim_description(project_media: create_project_media(team: t)) + assert_equal 3, t.filtered_fact_checks.count + assert_equal 3, t.filtered_fact_checks(text: 'Test').count + assert_equal 2, t.filtered_fact_checks(text: 'Foo Bar').count + assert_equal 1, t.filtered_fact_checks(text: 'Foo Bar Bravo').count + assert_equal 1, t.filtered_fact_checks(text: 'Foo Bar Alpha').count + end end From 1c75d6bebcc8be0e081538f0364801f9f36cc731 Mon Sep 17 00:00:00 2001 From: Sawy Date: Wed, 28 Aug 2024 13:00:05 +0300 Subject: [PATCH 2/5] CV2-5129: apply same search for Explainer --- app/models/team.rb | 6 +++++- test/models/team_2_test.rb | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/app/models/team.rb b/app/models/team.rb index 13695de9db..bb96e63aa1 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -495,7 +495,11 @@ def filtered_explainers(filters = {}) query = query.where(updated_at: Range.new(*format_times_search_range_filter(JSON.parse(filters[:updated_at]), nil))) unless filters[:updated_at].blank? # Filter by text - query = query.where('(title ILIKE ? OR url ILIKE ? OR description ILIKE ?)', *["%#{filters[:text]}%"]*3) if filters[:text].to_s.size > 2 + if filters[:text].to_s.size > 2 + tsquery = Team.sanitize_sql_array(["to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages + tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(url, '') || coalesce(description, ''))" + query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) + end # Exclude the ones already applied to a target item target = ProjectMedia.find_by_id(filters[:target_id].to_i) diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index f9bceb24bc..49e1ae56d5 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -1528,9 +1528,10 @@ def setup assert_equal ['none', 'link_preview'], t.available_newsletter_header_types end - test "should search for fact-checks by keywords" do + test "should search for fact-checks and explainers by keywords" do Sidekiq::Testing.fake! t = create_team + # Fact-checks create_fact_check title: 'Some Other Test', claim_description: create_claim_description(project_media: create_project_media(team: t)) create_fact_check title: 'Bar Bravo Foo Test', claim_description: create_claim_description(project_media: create_project_media(team: t)) create_fact_check title: 'Foo Alpha Bar Test', claim_description: create_claim_description(project_media: create_project_media(team: t)) @@ -1539,5 +1540,14 @@ def setup assert_equal 2, t.filtered_fact_checks(text: 'Foo Bar').count assert_equal 1, t.filtered_fact_checks(text: 'Foo Bar Bravo').count assert_equal 1, t.filtered_fact_checks(text: 'Foo Bar Alpha').count + # Explainer + create_explainer title: 'Some Other Test', team: t + create_explainer title: 'Bar Bravo Foo Test', team: t + create_explainer title: 'Foo Alpha Bar Test', team: t + assert_equal 3, t.filtered_explainers.count + assert_equal 3, t.filtered_explainers(text: 'Test').count + assert_equal 2, t.filtered_explainers(text: 'Foo Bar').count + assert_equal 1, t.filtered_explainers(text: 'Foo Bar Bravo').count + assert_equal 1, t.filtered_explainers(text: 'Foo Bar Alpha').count end end From 50126949ba3dfd39bf6caf2792e1d40fd80f6281 Mon Sep 17 00:00:00 2001 From: Sawy Date: Wed, 28 Aug 2024 13:11:48 +0300 Subject: [PATCH 3/5] CV2-5129: apply PR comment --- app/models/team.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/team.rb b/app/models/team.rb index bb96e63aa1..64d6001bd7 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -496,7 +496,7 @@ def filtered_explainers(filters = {}) # Filter by text if filters[:text].to_s.size > 2 - tsquery = Team.sanitize_sql_array(["to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages + tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(url, '') || coalesce(description, ''))" query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) end @@ -540,7 +540,7 @@ def filtered_fact_checks(filters = {}) # Filter by text if filters[:text].to_s.size > 2 - tsquery = Team.sanitize_sql_array(["to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages + tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || coalesce(url, ''))" query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) end From 0aef3e99ae7033401aa81ed6e16cf9aa1c759ceb Mon Sep 17 00:00:00 2001 From: computermacgyver Date: Wed, 28 Aug 2024 19:55:46 +0900 Subject: [PATCH 4/5] No need to split, strip, and join with websearch_to_tsquery --- app/models/team.rb | 4 ++-- test/models/team_2_test.rb | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/team.rb b/app/models/team.rb index 64d6001bd7..0768a80990 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -496,7 +496,7 @@ def filtered_explainers(filters = {}) # Filter by text if filters[:text].to_s.size > 2 - tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages + tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]]) # FIXME: May not work for all languages tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(url, '') || coalesce(description, ''))" query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) end @@ -540,7 +540,7 @@ def filtered_fact_checks(filters = {}) # Filter by text if filters[:text].to_s.size > 2 - tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text].split(/\s+/).map(&:strip).join(' & ')]) # FIXME: May not work for all languages + tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]]) # FIXME: May not work for all languages tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || coalesce(url, ''))" query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) end diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index 49e1ae56d5..1542c9491e 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -1540,6 +1540,7 @@ def setup assert_equal 2, t.filtered_fact_checks(text: 'Foo Bar').count assert_equal 1, t.filtered_fact_checks(text: 'Foo Bar Bravo').count assert_equal 1, t.filtered_fact_checks(text: 'Foo Bar Alpha').count + assert_equal 0, t.filtered_fact_checks(text: 'Foo Bar Delta').count # Explainer create_explainer title: 'Some Other Test', team: t create_explainer title: 'Bar Bravo Foo Test', team: t @@ -1549,5 +1550,6 @@ def setup assert_equal 2, t.filtered_explainers(text: 'Foo Bar').count assert_equal 1, t.filtered_explainers(text: 'Foo Bar Bravo').count assert_equal 1, t.filtered_explainers(text: 'Foo Bar Alpha').count + assert_equal 0, t.filtered_fact_checks(text: 'Foo Bar Delta').count end end From 902189ecb988c0e1214a03386dcf693bfdbb51c4 Mon Sep 17 00:00:00 2001 From: Sawy Date: Wed, 28 Aug 2024 14:05:01 +0300 Subject: [PATCH 5/5] CV2-5129: fix CC --- app/models/team.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/app/models/team.rb b/app/models/team.rb index 0768a80990..a3d8572e51 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -495,11 +495,7 @@ def filtered_explainers(filters = {}) query = query.where(updated_at: Range.new(*format_times_search_range_filter(JSON.parse(filters[:updated_at]), nil))) unless filters[:updated_at].blank? # Filter by text - if filters[:text].to_s.size > 2 - tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]]) # FIXME: May not work for all languages - tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(url, '') || coalesce(description, ''))" - query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) - end + query = self.filter_by_keywords(query, filters, 'Explainer') if filters[:text].to_s.size > 2 # Exclude the ones already applied to a target item target = ProjectMedia.find_by_id(filters[:target_id].to_i) @@ -539,11 +535,7 @@ def filtered_fact_checks(filters = {}) query = query.where('fact_checks.report_status' => filters[:report_status].to_a.map(&:to_s)) unless filters[:report_status].blank? # Filter by text - if filters[:text].to_s.size > 2 - tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]]) # FIXME: May not work for all languages - tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || coalesce(url, ''))" - query = query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) - end + query = self.filter_by_keywords(query, filters) if filters[:text].to_s.size > 2 # Exclude the ones already applied to a target item target = ProjectMedia.find_by_id(filters[:target_id].to_i) @@ -552,6 +544,16 @@ def filtered_fact_checks(filters = {}) query end + def filter_by_keywords(query, filters, type = 'FactCheck') + tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]]) # FIXME: May not work for all languages + if type == 'FactCheck' + tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || coalesce(url, ''))" + else + tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(description, '') || coalesce(url, ''))" + end + query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) + end + # private # # Please add private methods to app/models/concerns/team_private.rb