CV2-5011 refactors for making alegre dual purpose on text encoding

.github/workflows/ci-tests.yml

@@ -58,22 +58,22 @@ jobs:
 
     - name: Build Docker Container
       run: |
-        docker-compose build
+        docker compose build
     - name: Run syntax checks
       run: |
-        docker-compose run api bash -c 'touch /tmp/no-syntax-errors && find app lib config -name *.rb -exec bash -c "ruby -c {} >/dev/null || rm /tmp/no-syntax-errors" ";" && ls /tmp/no-syntax-errors'
+        docker compose run api bash -c 'touch /tmp/no-syntax-errors && find app lib config -name *.rb -exec bash -c "ruby -c {} >/dev/null || rm /tmp/no-syntax-errors" ";" && ls /tmp/no-syntax-errors'
     - name: Run Container
-      run: docker-compose -f docker-compose.yml -f docker-test.yml up -d
+      run: docker compose -f docker-compose.yml -f docker-test.yml up -d
 
     - name: Wait for the server to be ready
       run: |
         tail -f log/test.log &
         until curl --silent -I -f --fail http://localhost:3000 ; do printf .; sleep 1; done
     - name: Set up parallel environment
-      run: docker-compose exec -T api test/setup-parallel-env.sh
+      run: docker compose exec -T api test/setup-parallel-env.sh
 
     - name: Precompile Assets
-      run: docker-compose exec -T api bundle exec rake assets:precompile
+      run: docker compose exec -T api bundle exec rake assets:precompile
 
     - name: Prepare Parallel Runtime Log
       run: |
@@ -86,7 +86,7 @@ jobs:
       env:
         TEST_RETRY_COUNT: 3
       run: |
-        docker-compose exec -e TEST_RETRY_COUNT=$TEST_RETRY_COUNT -T -e PATTERN='models mailers integration workers lib contract' api test/run-tests.sh
+        docker compose exec -e TEST_RETRY_COUNT=$TEST_RETRY_COUNT -T -e PATTERN='models mailers integration workers lib contract' api test/run-tests.sh
 
     - name: After Functional Test
       env:
@@ -103,7 +103,7 @@ jobs:
         ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}
         AWS_REGION: eu-west-1
       run:  |
-        docker-compose exec -T -e GIT_BRANCH=$GIT_BRANCH \
+        docker compose exec -T -e GIT_BRANCH=$GIT_BRANCH \
         -e GITHUB_EVENT=$GITHUB_EVENT \
         -e GITHUB_TEST_RESULT=$GITHUB_TEST_RESULT \
         -e GITHUB_REPO=$GITHUB_REPO \
@@ -161,22 +161,22 @@ jobs:
 
     - name: Build Docker Container
       run: |
-        docker-compose build
+        docker compose build
     - name: Run syntax checks
       run: |
-        docker-compose run api bash -c 'touch /tmp/no-syntax-errors && find app lib config -name *.rb -exec bash -c "ruby -c {} >/dev/null || rm /tmp/no-syntax-errors" ";" && ls /tmp/no-syntax-errors'
+        docker compose run api bash -c 'touch /tmp/no-syntax-errors && find app lib config -name *.rb -exec bash -c "ruby -c {} >/dev/null || rm /tmp/no-syntax-errors" ";" && ls /tmp/no-syntax-errors'
     - name: Run Container
-      run: docker-compose -f docker-compose.yml -f docker-test.yml up -d
+      run: docker compose -f docker-compose.yml -f docker-test.yml up -d
 
     - name: Wait for the server to be ready
       run: |
         tail -f log/test.log &
         until curl --silent -I -f --fail http://localhost:3000 ; do printf .; sleep 1; done
     - name: Set up parallel environment
-      run: docker-compose exec -T api test/setup-parallel-env.sh
+      run: docker compose exec -T api test/setup-parallel-env.sh
 
     - name: Precompile Assets
-      run: docker-compose exec -T api bundle exec rake assets:precompile
+      run: docker compose exec -T api bundle exec rake assets:precompile
 
     - name: Prepare Parallel Runtime Log
       run: |
@@ -189,7 +189,7 @@ jobs:
       env:
         TEST_RETRY_COUNT: 3
       run: |
-        docker-compose exec -e TEST_RETRY_COUNT=$TEST_RETRY_COUNT -T -e PATTERN='controllers contract' api test/run-tests.sh
+        docker compose exec -e TEST_RETRY_COUNT=$TEST_RETRY_COUNT -T -e PATTERN='controllers contract' api test/run-tests.sh
 
     - name: After Unit Test
       env:
@@ -206,7 +206,7 @@ jobs:
         ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}
         AWS_REGION: eu-west-1
       run:  |
-        docker-compose exec -T -e GIT_BRANCH=$GIT_BRANCH \
+        docker compose exec -T -e GIT_BRANCH=$GIT_BRANCH \
         -e GITHUB_EVENT=$GITHUB_EVENT \
         -e GITHUB_TEST_RESULT=$GITHUB_TEST_RESULT \
         -e GITHUB_REPO=$GITHUB_REPO \
@@ -264,25 +264,25 @@ jobs:
 
     - name: Build Docker Container
       run: |
-        docker-compose build
+        docker compose build
 
     - name: Run syntax checks
       run: |
-        docker-compose run api bash -c 'touch /tmp/no-syntax-errors && find app lib config -name *.rb -exec bash -c "ruby -c {} >/dev/null || rm /tmp/no-syntax-errors" ";" && ls /tmp/no-syntax-errors'
+        docker compose run api bash -c 'touch /tmp/no-syntax-errors && find app lib config -name *.rb -exec bash -c "ruby -c {} >/dev/null || rm /tmp/no-syntax-errors" ";" && ls /tmp/no-syntax-errors'
 
     - name: Run Container
-      run: docker-compose -f docker-compose.yml -f docker-test.yml up -d
+      run: docker compose -f docker-compose.yml -f docker-test.yml up -d
 
     - name: Wait for the server to be ready
       run: |
         tail -f log/test.log &
         until curl --silent -I -f --fail http://localhost:3000 ; do printf .; sleep 1; done
 
     - name: Set up Parallel Environment
-      run: docker-compose exec -T api test/setup-parallel-env.sh
+      run: docker compose exec -T api test/setup-parallel-env.sh
 
     - name: Precompile Assets
-      run: docker-compose exec -T api bundle exec rake assets:precompile
+      run: docker compose exec -T api bundle exec rake assets:precompile
 
     - name: Prepare Parallel Runtime Log
       run: |
@@ -295,7 +295,7 @@ jobs:
       env:
         TEST_RETRY_COUNT: 3
       run: |
-        docker-compose exec -e TEST_RETRY_COUNT=$TEST_RETRY_COUNT -T -e PATTERN='controllers models mailers integration workers lib' api test/run-tests.sh
+        docker compose exec -e TEST_RETRY_COUNT=$TEST_RETRY_COUNT -T -e PATTERN='controllers models mailers integration workers lib' api test/run-tests.sh
 
     - name: After Contract Test
       env:
@@ -312,7 +312,7 @@ jobs:
         ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}
         AWS_REGION: eu-west-1
       run:  |
-        docker-compose exec -T -e GIT_BRANCH=$GIT_BRANCH \
+        docker compose exec -T -e GIT_BRANCH=$GIT_BRANCH \
         -e GITHUB_EVENT=$GITHUB_EVENT \
         -e GITHUB_TEST_RESULT=$GITHUB_TEST_RESULT \
         -e GITHUB_REPO=$GITHUB_REPO  \
@@ -324,4 +324,4 @@ jobs:
         -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
         -e ROLE_TO_ASSUME=$AWS_ROLE_TO_ASSUME \
         -e AWS_REGION=$AWS_REGION \
-        -e AWS_CONFIG_FILE=/app/credentials api test/test-coverage.sh
+        -e AWS_CONFIG_FILE=/app/credentials api test/test-coverage.sh

Gemfile.lock

@@ -726,8 +726,8 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
-    rexml (3.2.8)
-      strscan (>= 3.0.9)
+    rexml (3.3.3)
+      strscan
     rotp (6.3.0)
     rqrcode (2.1.1)
       chunky_png (~> 1.0)

app/graph/types/claim_description_type.rb

@@ -8,6 +8,7 @@ class ClaimDescriptionType < DefaultObject
   field :context, GraphQL::Types::String, null: true, resolver_method: :claim_context
   field :user, UserType, null: true
   field :project_media, ProjectMediaType, null: true
+  field :project_media_was, ProjectMediaType, null: true
   field :fact_check, FactCheckType, null: true do
     argument :report_status, GraphQL::Types::String, required: false, camelize: false
   end

app/graph/types/project_media_type.rb

@@ -320,6 +320,15 @@ def suggested_main_item
       &.source
   end
 
+  field :suggested_main_relationship, RelationshipType, null: true
+
+  def suggested_main_relationship
+    Relationship
+      .where("relationship_type = ?", Relationship.suggested_type.to_yaml)
+      .where(target_id: object.id)
+      .first
+  end
+
   field :confirmed_similar_relationships, RelationshipType.connection_type, null: true
 
   def confirmed_similar_relationships

app/graph/types/query_type.rb

@@ -235,6 +235,8 @@ def feed_team(id: nil, feed_id: nil, team_slug: nil)
     feed
     request
     tipline_message
+    fact_check
+    explainer
   ].each do |type|
     field type,
           "#{type.to_s.camelize}Type",

app/models/annotations/tag.rb

@@ -89,7 +89,7 @@ def self.run_bulk_create_callbacks(ids_json, pmids_json)
   def get_tag_text_reference
     if self.tag.is_a?(String)
       team_id = self.team&.id
-      tag_text = TagText.where(text: self.tag, team_id: team_id).last
+      tag_text = TagText.where(text: self.tag.strip, team_id: team_id).last
       if tag_text.nil? && team_id.present?
         tag_text = TagText.new
         tag_text.text = self.tag

app/models/bot/alegre.rb

@@ -611,7 +611,7 @@ def self.is_suggested_to_trash(source, target, relationship_type)
   end
 
   def self.create_relationship(source, target, pm_id_scores, relationship_type, original_source=nil, original_relationship_type=nil)
-    return nil if !self.can_create_relationship?(source, target, relationship_type)
+    return nil if source&.id == target&.id || !self.can_create_relationship?(source, target, relationship_type)
     r = Relationship.where(source_id: source.id, target_id: target.id)
     .where('relationship_type = ? OR relationship_type = ?', Relationship.confirmed_type.to_yaml, Relationship.suggested_type.to_yaml).last
     if r.nil?

app/models/claim_description.rb

@@ -38,6 +38,10 @@ def article_elasticsearch_data(action = 'create_or_update')
     self.index_in_elasticsearch(data)
   end
 
+  def project_media_was
+    ProjectMedia.find_by_id(self.project_media_id_before_last_save)
+  end
+
   private
 
   def set_team

app/models/concerns/alegre_v2.rb

@@ -158,6 +158,8 @@ def content_hash(project_media, field)
           return Rails.cache.read("url_sha:#{project_media.url}")
         elsif !project_media.is_text?
           return project_media.media.file.filename.split(".").first
+        else
+          return Digest::MD5.hexdigest(project_media.send(field).to_s)
         end
       end
     end
@@ -178,6 +180,26 @@ def delete_package(project_media, field, params={}, quiet=false)
       ).merge(params)
     end
 
+    def generic_package_text(project_media, field, params, fuzzy=false, match_across_content_types=true)
+      package = generic_package(project_media, field).merge(
+        params
+      ).merge(
+        models: self.indexing_models_to_use(project_media),
+        text: project_media.send(field),
+        fuzzy: fuzzy == 'true' || fuzzy.to_i == 1,
+        match_across_content_types: match_across_content_types,
+      )
+      team_id = project_media.team_id
+      language = self.language_for_similarity(team_id)
+      package[:language] = language if !language.nil?
+      package[:min_es_score] = self.get_min_es_score(team_id)
+      package
+    end
+
+    def delete_package_text(project_media, field, params)
+      generic_package_text(project_media, field, params)
+    end
+
     def generic_package_media(project_media, params)
       generic_package(project_media, nil).merge(
         url: media_file_url(project_media),
@@ -241,6 +263,10 @@ def store_package_audio(project_media, _field, params)
       generic_package_audio(project_media, params)
     end
 
+    def store_package_text(project_media, field, params)
+      generic_package_text(project_media, field, params)
+    end
+
     def get_sync(project_media, field=nil, params={})
       request_sync(
         store_package(project_media, field, params),

config/initializers/devise.rb

@@ -1,4 +1,5 @@
 require 'error_codes'
+require 'redis'
 
 class CustomFailure < Devise::FailureApp
   def respond
@@ -49,6 +50,12 @@ def http_auth_body
   end
   config.mailer = 'DeviseMailer'
   config.invite_for = 1.month
+
+  Warden::Manager.after_authentication do |user, auth, opts|
+    @redis = Redis.new(REDIS_CONFIG)
+    ip = auth.request.ip
+    @redis.decr("track:#{ip}")
+  end
 end
 
 AuthTrail.geocode = false

config/initializers/rack_attack.rb

@@ -36,6 +36,8 @@ def self.authenticated?(req)
         count = redis.incr("track:#{ip}")
         redis.expire("track:#{ip}", 3600) # Set the expiration time to 1 hour
 
+        redis.set("track:#{ip}", 0) if count < 0
+
         # Add IP to blocklist if count exceeds the threshold
         if count.to_i >= CheckConfig.get('login_block_limit', 100, :integer)
           redis.set("block:#{ip}", true)  # No expiration

lib/relay.idl

@@ -728,6 +728,7 @@ type ClaimDescription implements Node {
   id: ID!
   permissions: String
   project_media: ProjectMedia
+  project_media_was: ProjectMedia
   updated_at: String
   user: User
 }
@@ -11665,6 +11666,7 @@ type ProjectMedia implements Node {
   source_id: Int
   status: String
   suggested_main_item: ProjectMedia
+  suggested_main_relationship: Relationship
   suggested_similar_items_count: Int
   suggested_similar_relationships(
     """
@@ -11884,6 +11886,16 @@ type Query {
   bot_user(id: ID!): BotUser
   dynamic_annotation_field(only_cache: Boolean, query: String!): DynamicAnnotationField
 
+  """
+  Information about the explainer with given id
+  """
+  explainer(id: ID!): Explainer
+
+  """
+  Information about the fact_check with given id
+  """
+  fact_check(id: ID!): FactCheck
+
   """
   Information about the feed with given id
   """