diff --git a/.circleci/config.yml b/.circleci/config.yml index 43743f532..d612fdf63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ workflows: debian: buster matrix: parameters: - ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ] + ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ] # - test: # database: postgresql # sphinx_version: 3.3.1 @@ -62,7 +62,7 @@ workflows: debian: buster matrix: parameters: - ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ] + ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ] - test: name: "Manticore 3.5.4 with PostgreSQL" database: postgresql @@ -71,7 +71,7 @@ workflows: debian: buster matrix: parameters: - ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2' ] + ruby: [ '2.4.9', '2.5.8', '2.6.6', '2.7.2', '3.0.0' ] jobs: test: @@ -115,6 +115,8 @@ jobs: command: | if [ "<< parameters.ruby >>" == "2.7.2" ]; then export BUNDLER_VERSION=2.1.4 + elif [ "<< parameters.ruby >>" == "3.0.0" ]; then + export BUNDLER_VERSION=2.1.4 else export BUNDLER_VERSION=1.17.3 fi diff --git a/Appraisals b/Appraisals index 548fe8c4a..d61a10503 100644 --- a/Appraisals +++ b/Appraisals @@ -15,18 +15,18 @@ appraise 'rails_5_0' do gem 'jdbc-mysql', '~> 5.1.36', :platform => :jruby gem 'activerecord-jdbcmysql-adapter', '~> 50.0', :platform => :jruby gem 'activerecord-jdbcpostgresql-adapter', '~> 50.0', :platform => :jruby -end if RUBY_PLATFORM != "java" || ENV["SPHINX_VERSION"].to_f > 2.1 +end if (RUBY_PLATFORM != "java" || ENV["SPHINX_VERSION"].to_f > 2.1) && RUBY_VERSION.to_f < 3.0 appraise 'rails_5_1' do gem 'rails', '~> 5.1.0' gem 'mysql2', '~> 0.4.0', :platform => :ruby -end if RUBY_PLATFORM != 'java' +end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f < 3.0 appraise 'rails_5_2' do gem 'rails', '~> 5.2.0' gem 'mysql2', '~> 0.5.0', :platform => :ruby gem 'pg', '~> 1.0', :platform => :ruby -end if RUBY_PLATFORM != 'java' +end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f < 3.0 appraise 'rails_6_0' do gem 'rails', '~> 6.0.0' diff --git a/CHANGELOG.markdown b/CHANGELOG.markdown index 8629a7b5b..9f85326bf 100644 --- a/CHANGELOG.markdown +++ b/CHANGELOG.markdown @@ -2,6 +2,20 @@ All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file. +## 5.2.0 - 2021-06-12 + +[Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.2.0) + +### Added + +* Confirmed support for Ruby 3.0. +* Orphaned records in real-time indices can now be cleaned up without running `rails ts:rebuild`. Disabled by default, can be enabled by setting `real_time_tidy` to true per environment in `config/thinking_sphinx.yml` (and will need `ts:rebuild` to restructure indices upon initial deploy). More details in [#1192](https://github.com/pat/thinking-sphinx/pull/1192). + +### Fixed + +* Avoid loading ActiveRecord during Rails initialisation so app configuration can still have an impact ([@jdelStrother](https://github.com/jdelStrother) in [#1194](https://github.com/pat/thinking-sphinx/pull/1194)). +* Remove `app/indices` (in both the Rails app and engines) from Rails' eager load paths, which was otherwise leading to indices being loaded more than once. (See [#1191](https://github.com/pat/thinking-sphinx/issues/1191) and [#1195](https://github.com/pat/thinking-sphinx/issues/1195)). + ## 5.1.0 - 2020-12-28 [Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.1.0) diff --git a/README.textile b/README.textile index a72095ef3..f5c1ca0ac 100644 --- a/README.textile +++ b/README.textile @@ -1,6 +1,6 @@ h1. Thinking Sphinx -Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.1.0. +Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.2.0. h2. Upgrading @@ -14,7 +14,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp 
gem 'mysql2',          '~> 0.4',    :platform => :ruby
 gem 'jdbc-mysql',      '~> 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 5.1'
+gem 'thinking-sphinx', '~> 5.2' The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. @@ -29,18 +29,18 @@ h2. Requirements The current release of Thinking Sphinx works with the following versions of its dependencies: |_. Library |_. Minimum |_. Tested Against | -| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7 | -| Sphinx | v2.2.11 | v2.2.11, v3.2.1 | -| Manticore | v2.8 | v2.8, v3.4 | -| ActiveRecord | v4.2 | v4.2..v6.0 | +| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7, v3.0 | +| Sphinx | v2.2.11 | v2.2.11, v3.3.1 | +| Manticore | v2.8 | v2.8, v3.5 | +| ActiveRecord | v4.2 | v4.2..v6.1 | It _might_ work with older versions of Ruby, but it's highly recommended to update to a supported release. -It should also work with JRuby, but the test environment on Travis CI has been timing out, hence that's not actively tested against at the moment. +It should also work with JRuby, but the test environment for that in CI has been unreliable, hence that's not actively tested against at the moment. h3. Sphinx or Manticore -Thinking Sphinx is currently built for Sphinx 2.2.11 or newer (though it'll likely work with 2.1.x releases), or Manticore v2.8+. +Thinking Sphinx is currently built for Sphinx 2.2.11 or newer, or Manticore v2.8+. h3. Rails and ActiveRecord diff --git a/lib/thinking_sphinx.rb b/lib/thinking_sphinx.rb index 15bc37d4e..2e59e8c66 100644 --- a/lib/thinking_sphinx.rb +++ b/lib/thinking_sphinx.rb @@ -96,12 +96,15 @@ module Subscribers; end require 'thinking_sphinx/utf8' require 'thinking_sphinx/wildcard' # Extended -require 'thinking_sphinx/active_record' require 'thinking_sphinx/deltas' require 'thinking_sphinx/distributed' require 'thinking_sphinx/logger' require 'thinking_sphinx/real_time' -require 'thinking_sphinx/railtie' if defined?(Rails::Railtie) +if defined?(Rails::Railtie) + require 'thinking_sphinx/railtie' +else + require 'thinking_sphinx/active_record' +end ThinkingSphinx.before_index_hooks << ThinkingSphinx::Hooks::GuardPresence diff --git a/lib/thinking_sphinx/railtie.rb b/lib/thinking_sphinx/railtie.rb index 8ba1f8583..668a0e897 100644 --- a/lib/thinking_sphinx/railtie.rb +++ b/lib/thinking_sphinx/railtie.rb @@ -7,6 +7,7 @@ class ThinkingSphinx::Railtie < Rails::Railtie initializer 'thinking_sphinx.initialisation' do ActiveSupport.on_load(:active_record) do + require 'thinking_sphinx/active_record' ActiveRecord::Base.include ThinkingSphinx::ActiveRecord::Base end @@ -16,6 +17,10 @@ class ThinkingSphinx::Railtie < Rails::Railtie Rails.root.join("app", "indices").to_s ) end + + Rails.application.config.eager_load_paths -= + ThinkingSphinx::Configuration.instance.index_paths + Rails.application.config.eager_load_paths.freeze end end diff --git a/lib/thinking_sphinx/real_time/index/template.rb b/lib/thinking_sphinx/real_time/index/template.rb index 9b7dc635f..c8db1067d 100644 --- a/lib/thinking_sphinx/real_time/index/template.rb +++ b/lib/thinking_sphinx/real_time/index/template.rb @@ -13,6 +13,10 @@ def apply add_attribute primary_key, :sphinx_internal_id, :bigint add_attribute class_column, :sphinx_internal_class, :string, :facet => true add_attribute 0, :sphinx_deleted, :integer + + if tidying? + add_attribute -> (_) { Time.current.to_i }, :sphinx_updated_at, :timestamp + end end private @@ -34,7 +38,15 @@ def class_column [:class, :name] end + def config + ThinkingSphinx::Configuration.instance + end + def primary_key index.primary_key.to_sym end + + def tidying? + config.settings["real_time_tidy"] + end end diff --git a/lib/thinking_sphinx/real_time/populator.rb b/lib/thinking_sphinx/real_time/populator.rb index 2bfaa088b..b739445ea 100644 --- a/lib/thinking_sphinx/real_time/populator.rb +++ b/lib/thinking_sphinx/real_time/populator.rb @@ -7,6 +7,7 @@ def self.populate(index) def initialize(index) @index = index + @started_at = Time.current end def populate @@ -17,12 +18,14 @@ def populate instrument 'populated', :instances => instances end + transcriber.clear_before(started_at) if configuration.settings["real_time_tidy"] + instrument 'finish_populating' end private - attr_reader :index + attr_reader :index, :started_at delegate :controller, :batch_size, :to => :configuration delegate :scope, :to => :index diff --git a/lib/thinking_sphinx/real_time/transcriber.rb b/lib/thinking_sphinx/real_time/transcriber.rb index dc8bbc02a..4562bb3c5 100644 --- a/lib/thinking_sphinx/real_time/transcriber.rb +++ b/lib/thinking_sphinx/real_time/transcriber.rb @@ -5,6 +5,12 @@ def initialize(index) @index = index end + def clear_before(time) + execute <<~SQL.strip + DELETE FROM #{@index.name} WHERE sphinx_updated_at < #{time.to_i} + SQL + end + def copy(*instances) items = instances.select { |instance| instance.persisted? && copy?(instance) diff --git a/lib/thinking_sphinx/real_time/translator.rb b/lib/thinking_sphinx/real_time/translator.rb index 20e6da3de..f723c762d 100644 --- a/lib/thinking_sphinx/real_time/translator.rb +++ b/lib/thinking_sphinx/real_time/translator.rb @@ -10,6 +10,7 @@ def initialize(object, column) end def call + return name.call(object) if name.is_a?(Proc) return name unless name.is_a?(Symbol) return result unless result.is_a?(String) diff --git a/lib/thinking_sphinx/settings.rb b/lib/thinking_sphinx/settings.rb index 402042d51..850150a29 100644 --- a/lib/thinking_sphinx/settings.rb +++ b/lib/thinking_sphinx/settings.rb @@ -19,7 +19,8 @@ class ThinkingSphinx::Settings "binlog_path" => "tmp/binlog/ENVIRONMENT", "workers" => "threads", "mysql_encoding" => "utf8", - "maximum_statement_length" => (2 ** 23) - 5 + "maximum_statement_length" => (2 ** 23) - 5, + "real_time_tidy" => false }.freeze def self.call(configuration) diff --git a/spec/thinking_sphinx/real_time/index_spec.rb b/spec/thinking_sphinx/real_time/index_spec.rb index ffac6960a..06e1e152d 100644 --- a/spec/thinking_sphinx/real_time/index_spec.rb +++ b/spec/thinking_sphinx/real_time/index_spec.rb @@ -5,7 +5,9 @@ describe ThinkingSphinx::RealTime::Index do let(:index) { ThinkingSphinx::RealTime::Index.new :user } let(:config) { double('config', :settings => {}, - :indices_location => 'location', :next_offset => 8) } + :indices_location => 'location', :next_offset => 8, + :index_set_class => index_set_class) } + let(:index_set_class) { double(:index_set_class, :reference_name => :user) } before :each do allow(ThinkingSphinx::Configuration).to receive_messages :instance => config @@ -61,6 +63,16 @@ it "has the internal deleted attribute by default" do expect(index.attributes.collect(&:name)).to include('sphinx_deleted') end + + it "does not have an internal updated_at attribute by default" do + expect(index.attributes.collect(&:name)).to_not include('sphinx_updated_at') + end + + it "has an internal updated_at attribute if real_time_tidy is true" do + config.settings["real_time_tidy"] = true + + expect(index.attributes.collect(&:name)).to include('sphinx_updated_at') + end end describe '#delta?' do diff --git a/thinking-sphinx.gemspec b/thinking-sphinx.gemspec index e41a86832..f08fefb10 100644 --- a/thinking-sphinx.gemspec +++ b/thinking-sphinx.gemspec @@ -5,7 +5,7 @@ $:.push File.expand_path('../lib', __FILE__) Gem::Specification.new do |s| s.name = 'thinking-sphinx' - s.version = '5.1.0' + s.version = '5.2.0' s.platform = Gem::Platform::RUBY s.authors = ["Pat Allan"] s.email = ["pat@freelancing-gods.com"]