diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index b0239bd87b..de219a6db4 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -283,7 +283,6 @@ def get_target_field(project_media, field) end def parse_similarity_results(project_media, field, results, relationship_type) - results ||= [] Hash[results.collect{|result| result["context"] = isolate_relevant_context(project_media, result) [ diff --git a/doc/api-user.md b/doc/api-user.md index 89bbaa10ae..7d462dba5a 100644 --- a/doc/api-user.md +++ b/doc/api-user.md @@ -269,36 +269,6 @@ As we can see above, the `geolocation` task response should be a valid [GeoJSON] The `url` and `quote` can't be both defined at the same time. It's one or the other. If the `quote` field is set, it means you're creating a claim. If the `url` field is set, it means you're creating a report of type "link". -## Create original claim when creating media - -When using `createProjectMedia` to create new project media, you can use the `set_original_claim` field to import a URL as the original claim. This URL can be in the following formats: - -1. Video URL: This will create a video as the original claim. -2. Audio URL: This will create an audio as the original claim. -3. Image URL: This will create a picture/image as the original claim. -4. Normal URL: A Link will be created as the original claim -5. If the value in `set_original_claim` is not a URL, then the original claim will be created in plain-text. - -Here is an example of how to use this field: - -_Query_ -```graphql -mutation { - createProjectMedia(input: { - project_id: 4, - set_original_claim: "https://someurl.com/somefile.mp3" - }) { - project_media { - title - type - url - } - } -} -``` - -This example will create an audio as the original claim. - ## Updating media We can update media using the GraphQL (Base 64) id. In the example below, we change the description of a media: diff --git a/lib/rss_feed.rb b/lib/rss_feed.rb index 76255bc003..767a0eb413 100644 --- a/lib/rss_feed.rb +++ b/lib/rss_feed.rb @@ -18,9 +18,7 @@ def get_articles(count = 3) feed = RSS::Parser.parse(rss, false) unless feed.nil? feed.items.first(count).each do |item| - title = item.title.kind_of?(String) ? item.title : item.title.content - link = item.link.kind_of?(String) ? item.link : item.link.href - output << title.to_s.strip + "\n" + link.to_s.strip + output << item.title.strip + "\n" + item.link.strip end end end diff --git a/lib/tasks/data/statistics.rake b/lib/tasks/data/statistics.rake index a1c5c1a3f7..d01967c16b 100644 --- a/lib/tasks/data/statistics.rake +++ b/lib/tasks/data/statistics.rake @@ -28,7 +28,11 @@ namespace :check do team = Team.find(team_id) languages = team.get_languages.to_a - platforms = bot.user == BotUser.smooch_user ? bot.smooch_enabled_integrations.keys : Bot::Smooch::SUPPORTED_INTEGRATION_NAMES.keys + if bot.user == BotUser.smooch_user + platforms = bot.smooch_enabled_integrations.keys + else + platforms = Bot::Smooch::SUPPORTED_INTEGRATION_NAMES.keys + end team_stats = Hash.new(0) puts "[#{Time.now}] Generating month tipline statistics for team with ID #{team_id}. (#{index + 1} / #{team_ids.length})" @@ -97,39 +101,40 @@ namespace :check do raise Check::Statistics::IncompleteRunError.new("Failed to calculate #{errors.length} monthly team statistics") if errors.any? end - # bundle exec rake check:data:regenerate_statistics[start_date] - desc 'Regenerate all historic statistics for all workspaces from a given start date' - task :regenerate_statistics, [:start_date] => [:environment] do |_t, args| + # bundle exec rake check:data:regenerate_statistics[unique_newsletters_sent] + desc 'Regenerate specified historic statistic for all workspaces' + task :regenerate_statistics, [:stats_to_generate] => [:environment] do |_t, args| old_logger = ActiveRecord::Base.logger ActiveRecord::Base.logger = nil - start_date = DateTime.parse(args.start_date) rescue nil - if start_date.nil? - $stderr.puts "Invalid or missing start_date argument" - raise Check::Statistics::ArgumentError.new("Invalid or missing start_date argument") - end - - puts "[#{Time.now}] Starting to regenerate all statistics from #{start_date}" + puts "[#{Time.now}] Attempting to regenerate keys: #{args.stats_to_generate}" begin + # Give user help if they want it supported_stats = %w( unique_newsletters_sent ) - puts "[#{Time.now}] Regenerating stats for the following keys: #{supported_stats}. Total to update: #{MonthlyTeamStatistic.where('start_date >= ?', start_date).count}" + # Make sure we have at least one valid argument + requested_stats = (args.stats_to_generate || '').split(',').map(&:strip) + valid_requested_stats = requested_stats.intersection(supported_stats) + unless valid_requested_stats.length > 0 + raise Check::Statistics::ArgumentError.new("Argument '#{args.stats_to_generate}' is invalid. We currently support the following values passed a comma-separated list: #{supported_stats.join(',')}.") + end + + puts "[#{Time.now}] Regenerating stats for the following keys: #{valid_requested_stats}. Total to update: #{MonthlyTeamStatistic.count}" + # Update all of the stats total_successful = Hash.new(0) - MonthlyTeamStatistic.where('start_date >= ?', start_date).find_each do |monthly_stats| + MonthlyTeamStatistic.find_each do |monthly_stats| team_id = monthly_stats.team_id start_date = monthly_stats.start_date end_date = monthly_stats.end_date language = monthly_stats.language begin - supported_stats.each do |stat| - method_name = :number_of_newsletters_sent - result = CheckStatistics.send(method_name, team_id, start_date, end_date, language) - monthly_stats.update!(stat => result) - total_successful[stat.to_sym] += 1 + if valid_requested_stats.include?('unique_newsletters_sent') + monthly_stats.update!(unique_newsletters_sent: CheckStatistics.number_of_newsletters_sent(team_id, start_date, end_date, language)) + total_successful[:unique_newsletters_sent] += 1 end rescue StandardError => e $stderr.puts "[#{Time.now}] Failed to update MonthlyTeamStatistic with ID #{monthly_stats.id}. Error: #{e}" @@ -139,6 +144,7 @@ namespace :check do puts "[#{Time.now}] Finished updating MonthlyTeamStatistics. Total updated: #{total_successful}" rescue StandardError => e $stderr.puts e + next ensure ActiveRecord::Base.logger = old_logger end diff --git a/test/lib/tasks/statistics_test.rb b/test/lib/tasks/statistics_test.rb index d00519537c..ddb8a1137b 100644 --- a/test/lib/tasks/statistics_test.rb +++ b/test/lib/tasks/statistics_test.rb @@ -337,7 +337,9 @@ def teardown test "check:data:statistics allows generating conversations for months before april 1 2023, with argument" do date = DateTime.new(2023,01,01) + create_project_media(user: BotUser.smooch_user, team: @tipline_team, created_at: date + 2.weeks) + CheckStatistics.stubs(:get_statistics).returns( { platform: 'whatsapp', @@ -346,93 +348,120 @@ def teardown end_date: date, } ) - + travel_to DateTime.new(2023,01,01) + out, err = capture_io do # pass in ignore_convo_cutoff: true Rake::Task['check:data:statistics'].invoke(true) end Rake::Task['check:data:statistics'].reenable - + conversations = MonthlyTeamStatistic.where(team: @tipline_team).pluck(:conversations_24hr).uniq assert_equal 1, conversations.count assert !conversations.first.nil? end - - test "check:data:regenerate_statistics errors if start_date argument is invalid" do + + test "check:data:regenerate_statistics errors if only an unsupported argument is passed" do out, err = capture_io do - assert_raises(Check::Statistics::ArgumentError) do - Rake::Task['check:data:regenerate_statistics'].invoke("invalid_date") - end + Rake::Task['check:data:regenerate_statistics'].invoke("foo") end Rake::Task['check:data:regenerate_statistics'].reenable - - assert_match /Invalid or missing start_date argument/, err + + assert err.present? end - - test "check:data:regenerate_statistics regenerates stats from the provided start date" do - start_date = "2023-04-01" + + test "check:data:regenerate_statistics accepts arguments as comma separated list or string" do + out, err = capture_io do + Rake::Task['check:data:regenerate_statistics'].invoke("unique_newsletters_sent") + end + Rake::Task['check:data:regenerate_statistics'].reenable + + assert err.blank? + + out, err = capture_io do + Rake::Task['check:data:regenerate_statistics'].invoke("unique_newsletters_sent,foo") + end + Rake::Task['check:data:regenerate_statistics'].reenable + + assert err.blank? + end + + test "check:data:regenerate_statistics outputs supported arguments if no args provided" do + out, err = capture_io do + Rake::Task['check:data:regenerate_statistics'].invoke + end + Rake::Task['check:data:regenerate_statistics'].reenable + + assert_match /unique_newsletters_sent/, err + + out, err = capture_io do + Rake::Task['check:data:regenerate_statistics'].invoke + end + Rake::Task['check:data:regenerate_statistics'].reenable + + assert_match /unique_newsletters_sent/, err + end + + test "check:data:regenerate_statistics regenerates any monthly team statistics present in database for provided stat" do previous_month_start = DateTime.new(2023,4,1,0,0,0) previous_month_end = DateTime.new(2023,4,30,23,59,59) - + other_workspace_with_stats = create_team - + team_stat_one = create_monthly_team_statistic(team: @tipline_team, language: 'en', start_date: previous_month_start, end_date: previous_month_end) team_stat_two = create_monthly_team_statistic(team: @tipline_team, language: 'es', start_date: @start_of_month, end_date: @current_date) team_stat_three = create_monthly_team_statistic(team: other_workspace_with_stats, language: 'en', start_date: @start_of_month, end_date: @current_date) - + CheckStatistics.stubs(:number_of_newsletters_sent).with(@tipline_team.id, team_stat_one.start_date, team_stat_one.end_date, 'en').returns(100) CheckStatistics.expects(:number_of_newsletters_sent).with(@tipline_team.id, team_stat_two.start_date, team_stat_two.end_date, 'es').returns(300) CheckStatistics.expects(:number_of_newsletters_sent).with(other_workspace_with_stats.id, team_stat_three.start_date, team_stat_three.end_date, 'en').returns(400) travel_to @current_date - + out, err = capture_io do - Rake::Task['check:data:regenerate_statistics'].invoke(start_date) + Rake::Task['check:data:regenerate_statistics'].invoke("unique_newsletters_sent") end Rake::Task['check:data:regenerate_statistics'].reenable - assert err.blank? - + # en, previous month - stats_one = MonthlyTeamStatistic.find_by(team: @tipline_team, language: 'en', start_date: previous_month_start) + stats_one = MonthlyTeamStatistic.first assert_equal @tipline_team.id, stats_one.team_id - assert_equal previous_month_start.to_i, stats_one.start_date.to_i - assert_equal previous_month_end.to_i, stats_one.end_date.to_i + assert_equal (@current_date - 1.month).beginning_of_month.to_i, stats_one.start_date.to_i + assert_equal (@current_date - 1.month).end_of_month.to_i, stats_one.end_date.to_i assert_equal 'en', stats_one.language assert_equal 100, stats_one.unique_newsletters_sent - + # es, current month - stats_two = MonthlyTeamStatistic.find_by(team: @tipline_team, language: 'es', start_date: @start_of_month) + stats_two = MonthlyTeamStatistic.second assert_equal @tipline_team.id, stats_two.team_id assert_equal @start_of_month.to_i, stats_two.start_date.to_i assert_equal @current_date.to_i, stats_two.end_date.to_i assert_equal 'es', stats_two.language assert_equal 300, stats_two.unique_newsletters_sent - - # second workspace - en, current month - stats_three = MonthlyTeamStatistic.find_by(team: other_workspace_with_stats, language: 'en', start_date: @start_of_month) + + # second workspace - es, current month + stats_three = MonthlyTeamStatistic.third assert_equal other_workspace_with_stats.id, stats_three.team_id assert_equal @start_of_month.to_i, stats_three.start_date.to_i assert_equal @current_date.to_i, stats_three.end_date.to_i assert_equal 'en', stats_three.language assert_equal 400, stats_three.unique_newsletters_sent end - + test "check:data:regenerate_statistics doesn't explode if tipline has been disabled, and sets newsletters to nil" do - start_date = "2023-04-01" random_team = create_team create_monthly_team_statistic(team: random_team, language: 'es', start_date: @start_of_month, end_date: @current_date) - + travel_to @current_date - + out, err = capture_io do - Rake::Task['check:data:regenerate_statistics'].invoke(start_date) + Rake::Task['check:data:regenerate_statistics'].invoke("unique_newsletters_sent") end Rake::Task['check:data:regenerate_statistics'].reenable - assert err.blank? - + stats_one = MonthlyTeamStatistic.first assert_nil stats_one.unique_newsletters_sent - end + end end