Merge branch 'release/v5.2.0' into master

.circleci/config.yml

@@ -28,7 +28,7 @@ workflows:
           debian: buster
           matrix:
             parameters:
-              ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ]
+              ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ]
       # - test:
       #     database: postgresql
       #     sphinx_version: 3.3.1
@@ -62,7 +62,7 @@ workflows:
           debian: buster
           matrix:
             parameters:
-              ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ]
+              ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ]
       - test:
           name: "Manticore 3.5.4 with PostgreSQL"
           database: postgresql
@@ -71,7 +71,7 @@ workflows:
           debian: buster
           matrix:
             parameters:
-              ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ]
+              ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ]
 
 jobs:
   test:
@@ -115,6 +115,8 @@ jobs:
           command: |
             if [ "<< parameters.ruby >>" == "2.7.2" ]; then
               export BUNDLER_VERSION=2.1.4
+            elif [ "<< parameters.ruby >>" == "3.0.0" ]; then
+              export BUNDLER_VERSION=2.1.4
             else
               export BUNDLER_VERSION=1.17.3
             fi

Appraisals

@@ -15,18 +15,18 @@ appraise 'rails_5_0' do
   gem 'jdbc-mysql',                          '~> 5.1.36', :platform => :jruby
   gem 'activerecord-jdbcmysql-adapter',      '~> 50.0',   :platform => :jruby
   gem 'activerecord-jdbcpostgresql-adapter', '~> 50.0',   :platform => :jruby
-end if RUBY_PLATFORM != "java" || ENV["SPHINX_VERSION"].to_f > 2.1
+end if (RUBY_PLATFORM != "java" || ENV["SPHINX_VERSION"].to_f > 2.1) && RUBY_VERSION.to_f < 3.0
 
 appraise 'rails_5_1' do
   gem 'rails',  '~> 5.1.0'
   gem 'mysql2', '~> 0.4.0', :platform => :ruby
-end if RUBY_PLATFORM != 'java'
+end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f < 3.0
 
 appraise 'rails_5_2' do
   gem 'rails',  '~> 5.2.0'
   gem 'mysql2', '~> 0.5.0', :platform => :ruby
   gem 'pg',     '~> 1.0',   :platform => :ruby
-end if RUBY_PLATFORM != 'java'
+end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f < 3.0
 
 appraise 'rails_6_0' do
   gem 'rails',  '~> 6.0.0'

CHANGELOG.markdown

@@ -2,6 +2,20 @@
 
 All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file.
 
+## 5.2.0 - 2021-06-12
+
+[Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.2.0)
+
+### Added
+
+* Confirmed support for Ruby 3.0.
+* Orphaned records in real-time indices can now be cleaned up without running `rails ts:rebuild`. Disabled by default, can be enabled by setting `real_time_tidy` to true per environment in `config/thinking_sphinx.yml` (and will need `ts:rebuild` to restructure indices upon initial deploy). More details in [#1192](https://github.com/pat/thinking-sphinx/pull/1192).
+
+### Fixed
+
+* Avoid loading ActiveRecord during Rails initialisation so app configuration can still have an impact ([@jdelStrother](https://github.com/jdelStrother) in [#1194](https://github.com/pat/thinking-sphinx/pull/1194)).
+* Remove `app/indices` (in both the Rails app and engines) from Rails' eager load paths, which was otherwise leading to indices being loaded more than once. (See [#1191](https://github.com/pat/thinking-sphinx/issues/1191) and [#1195](https://github.com/pat/thinking-sphinx/issues/1195)).
+
 ## 5.1.0 - 2020-12-28
 
 [Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.1.0)

README.textile

@@ -1,6 +1,6 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.1.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.2.0.
 
 h2. Upgrading
 
@@ -14,7 +14,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp
 
 <pre><code>gem 'mysql2',          '~> 0.4',    :platform => :ruby
 gem 'jdbc-mysql',      '~> 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 5.1'</code></pre>
+gem 'thinking-sphinx', '~> 5.2'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database.
 
@@ -29,18 +29,18 @@ h2. Requirements
 The current release of Thinking Sphinx works with the following versions of its dependencies:
 
 |_. Library |_. Minimum |_. Tested Against |
-| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7 |
-| Sphinx | v2.2.11 | v2.2.11, v3.2.1 |
-| Manticore | v2.8 | v2.8, v3.4 |
-| ActiveRecord | v4.2 | v4.2..v6.0 |
+| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7, v3.0 |
+| Sphinx | v2.2.11 | v2.2.11, v3.3.1 |
+| Manticore | v2.8 | v2.8, v3.5 |
+| ActiveRecord | v4.2 | v4.2..v6.1 |
 
 It _might_ work with older versions of Ruby, but it's highly recommended to update to a supported release.
 
-It should also work with JRuby, but the test environment on Travis CI has been timing out, hence that's not actively tested against at the moment.
+It should also work with JRuby, but the test environment for that in CI has been unreliable, hence that's not actively tested against at the moment.
 
 h3. Sphinx or Manticore
 
-Thinking Sphinx is currently built for Sphinx 2.2.11 or newer (though it'll likely work with 2.1.x releases), or Manticore v2.8+.
+Thinking Sphinx is currently built for Sphinx 2.2.11 or newer, or Manticore v2.8+.
 
 h3. Rails and ActiveRecord
 

lib/thinking_sphinx.rb

@@ -96,12 +96,15 @@ module Subscribers; end
 require 'thinking_sphinx/utf8'
 require 'thinking_sphinx/wildcard'
 # Extended
-require 'thinking_sphinx/active_record'
 require 'thinking_sphinx/deltas'
 require 'thinking_sphinx/distributed'
 require 'thinking_sphinx/logger'
 require 'thinking_sphinx/real_time'
 
-require 'thinking_sphinx/railtie' if defined?(Rails::Railtie)
+if defined?(Rails::Railtie)
+  require 'thinking_sphinx/railtie'
+else
+  require 'thinking_sphinx/active_record'
+end
 
 ThinkingSphinx.before_index_hooks << ThinkingSphinx::Hooks::GuardPresence

lib/thinking_sphinx/railtie.rb

@@ -7,6 +7,7 @@ class ThinkingSphinx::Railtie < Rails::Railtie
 
   initializer 'thinking_sphinx.initialisation' do
     ActiveSupport.on_load(:active_record) do
+      require 'thinking_sphinx/active_record'
       ActiveRecord::Base.include ThinkingSphinx::ActiveRecord::Base
     end
 
@@ -16,6 +17,10 @@ class ThinkingSphinx::Railtie < Rails::Railtie
           Rails.root.join("app", "indices").to_s
         )
       end
+
+      Rails.application.config.eager_load_paths -=
+        ThinkingSphinx::Configuration.instance.index_paths
+      Rails.application.config.eager_load_paths.freeze
     end
   end
 

lib/thinking_sphinx/real_time/index/template.rb

@@ -13,6 +13,10 @@ def apply
     add_attribute primary_key,  :sphinx_internal_id,    :bigint
     add_attribute class_column, :sphinx_internal_class, :string, :facet => true
     add_attribute 0,            :sphinx_deleted,        :integer
+
+    if tidying?
+      add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp
+    end
   end
 
   private
@@ -34,7 +38,15 @@ def class_column
     [:class, :name]
   end
 
+  def config
+    ThinkingSphinx::Configuration.instance
+  end
+
   def primary_key
     index.primary_key.to_sym
   end
+
+  def tidying?
+    config.settings["real_time_tidy"]
+  end
 end

lib/thinking_sphinx/real_time/populator.rb

@@ -7,6 +7,7 @@ def self.populate(index)
 
   def initialize(index)
     @index = index
+    @started_at = Time.current
   end
 
   def populate
@@ -17,12 +18,14 @@ def populate
       instrument 'populated', :instances => instances
     end
 
+    transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"]
+
     instrument 'finish_populating'
   end
 
   private
 
-  attr_reader :index
+  attr_reader :index, :started_at
 
   delegate :controller, :batch_size, :to => :configuration
   delegate :scope,                   :to => :index

lib/thinking_sphinx/real_time/transcriber.rb

@@ -5,6 +5,12 @@ def initialize(index)
     @index = index
   end
 
+  def clear_before(time)
+    execute <<~SQL.strip
+      DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i}
+    SQL
+  end
+
   def copy(*instances)
     items = instances.select { |instance|
       instance.persisted? && copy?(instance)

lib/thinking_sphinx/real_time/translator.rb

@@ -10,6 +10,7 @@ def initialize(object, column)
   end
 
   def call
+    return name.call(object) if name.is_a?(Proc)
     return name   unless name.is_a?(Symbol)
     return result unless result.is_a?(String)
 

lib/thinking_sphinx/settings.rb

@@ -19,7 +19,8 @@ class ThinkingSphinx::Settings
     "binlog_path"              => "tmp/binlog/ENVIRONMENT",
     "workers"                  => "threads",
     "mysql_encoding"           => "utf8",
-    "maximum_statement_length" => (2 ** 23) - 5
+    "maximum_statement_length" => (2 ** 23) - 5,
+    "real_time_tidy"           => false
   }.freeze
 
   def self.call(configuration)

spec/thinking_sphinx/real_time/index_spec.rb

@@ -5,7 +5,9 @@
 describe ThinkingSphinx::RealTime::Index do
   let(:index)        { ThinkingSphinx::RealTime::Index.new :user }
   let(:config)       { double('config', :settings => {},
-    :indices_location => 'location', :next_offset => 8) }
+    :indices_location => 'location', :next_offset => 8,
+    :index_set_class => index_set_class) }
+  let(:index_set_class) { double(:index_set_class, :reference_name => :user) }
 
   before :each do
     allow(ThinkingSphinx::Configuration).to receive_messages :instance => config
@@ -61,6 +63,16 @@
     it "has the internal deleted attribute by default" do
       expect(index.attributes.collect(&:name)).to include('sphinx_deleted')
     end
+
+    it "does not have an internal updated_at attribute by default" do
+      expect(index.attributes.collect(&:name)).to_not include('sphinx_updated_at')
+    end
+
+    it "has an internal updated_at attribute if real_time_tidy is true" do
+      config.settings["real_time_tidy"] = true
+
+      expect(index.attributes.collect(&:name)).to include('sphinx_updated_at')
+    end
   end
 
   describe '#delta?' do

thinking-sphinx.gemspec

@@ -5,7 +5,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = 'thinking-sphinx'
-  s.version     = '5.1.0'
+  s.version     = '5.2.0'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Pat Allan"]
   s.email       = ["[email protected]"]