Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change to using published solrcloud gem #17

Merged
merged 3 commits into from
Dec 1, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Gemfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
source "https://rubygems.org"
git_source(:github) { |repo| "https://github.com/#{repo}.git" }

gem "httpx", "~> 0.21"
gem "httpx" # , "~> 0.21"
gem "zinzout", "~> 0.1"
gem "faraday", "~> 2.5"
gem "faraday-follow_redirects"
Expand All @@ -12,7 +12,7 @@ gem "canister"
gem "rubyzip"
gem "semantic_logger"
gem "thor"
gem "solr_cloud-connection", git: "https://github.com/mlibrary/solr_cloud-connection"
gem "solr_cloud-connection"

gem "sqlite3", "~> 1.4", platforms: :mri
gem "jdbc-sqlite3", "~> 3.28", platforms: :jruby
Expand Down
27 changes: 12 additions & 15 deletions Gemfile.lock
Original file line number Diff line number Diff line change
@@ -1,12 +1,3 @@
GIT
remote: https://github.com/mlibrary/solr_cloud-connection
revision: ea3c856c5611d5fe070677dd09b6b663bc5a2944
specs:
solr_cloud-connection (0.1.0)
faraday
httpx
rubyzip

GEM
remote: https://rubygems.org/
specs:
Expand All @@ -24,7 +15,7 @@ GEM
diff-lcs (1.5.0)
docile (1.4.0)
dotenv (2.8.1)
faraday (2.7.11)
faraday (2.7.12)
base64
faraday-net_http (>= 2.0, < 3.1)
ruby2_keywords (>= 0.0.4)
Expand All @@ -35,9 +26,9 @@ GEM
ffi-icu (0.5.1)
ffi (~> 1.0, >= 1.0.9)
hashdiff (1.0.1)
http-2-next (0.5.1)
httpx (0.24.7)
http-2-next (< 1.0.0)
http-2-next (1.0.1)
httpx (1.1.5)
http-2-next (>= 1.0.1)
json (2.6.3)
language_server-protocol (3.17.0.3)
lint_roller (1.1.0)
Expand Down Expand Up @@ -100,6 +91,11 @@ GEM
simplecov_json_formatter (~> 0.1)
simplecov-html (0.12.3)
simplecov_json_formatter (0.1.4)
solr_cloud-connection (0.1.0)
faraday (~> 2.7.12)
httpx (~> 1.1.5)
rubyzip (~> 2.3.0)
sqlite3 (1.6.7-arm64-darwin)
sqlite3 (1.6.7-x86_64-linux)
standard (1.31.2)
language_server-protocol (~> 3.17.0.2)
Expand All @@ -124,6 +120,7 @@ GEM
zinzout (0.1.1)

PLATFORMS
arm64-darwin-22
x86_64-linux

DEPENDENCIES
Expand All @@ -134,7 +131,7 @@ DEPENDENCIES
faraday (~> 2.5)
faraday-follow_redirects
ffi-icu
httpx (~> 0.21)
httpx
jdbc-sqlite3 (~> 3.28)
milemarker (~> 1.0)
mysql2
Expand All @@ -145,7 +142,7 @@ DEPENDENCIES
semantic_logger
sequel (~> 5.60)
simplecov
solr_cloud-connection!
solr_cloud-connection
sqlite3 (~> 1.4)
standardrb
thor
Expand Down
38 changes: 12 additions & 26 deletions lib/authority_browse/solr.rb
Original file line number Diff line number Diff line change
Expand Up @@ -75,28 +75,14 @@ def self.set_up_daily_collection
#
# @return[Nil]
def self.set_daily_reindex_alias
S.solrcloud.get(
"solr/admin/collections",
{
action: "CREATEALIAS",
name: reindex_alias,
collections: [collection_name]
}
)
S.solrcloud.create_alias(name: reindex_alias, collection_name: collection_name, force: true)
end

# This sets the production alias to today's collection.
#
# @return[Nil]
def self.set_production_alias
S.solrcloud.get(
"solr/admin/collections",
{
action: "CREATEALIAS",
name: production_alias,
collections: [collection_name]
}
)
S.solrcloud.create_alias(name: production_alias, collection_name: collection_name, force: true)
end

# This verifies that today's collection has enough documents in it. For now
Expand All @@ -112,25 +98,25 @@ def self.verify_reindex
# than the newest three authority_browse collections.
#
# @return[Nil]
def self.prune_old_collections
def self.prune_old_collections(keep: 3)
S.logger.info "Pruning the following collections: #{list_old_collections}"
list_old_collections.each do |coll|
S.solrcloud.get("/solr/admin/collections", {action: "DELETE", name: coll, wt: "json"})
list_old_collections(keep: keep).each do |coll|
coll.delete!
end
end

# Lists the authority_browse collections that are older than the newest
# three authority_browse collections
#
# @param list [Array] Array of all SolrCloud collections
# @return [Array] Array of old authority browse Solrcloud collection
# strings
def self.list_old_collections(list = S.solrcloud.collections)
# @param list [Array]<SolrCloud::Collection> Array of all SolrCloud collections
# @param keep [Integer] how many versions to keep, even if they're old
# @return [Array]<SolrCloud::Collection> Array of old authority browse Solrcloud collections
def self.list_old_collections(list: S.solrcloud.collections, keep: 3)
list.select do |item|
item.match?("authority_browse")
item.name.match?("authority_browse")
end.sort do |a, b|
Date.parse(a.split("_").last) <=> Date.parse(b.split("_").last)
end[0..-4]
a.name.split("_").last <=> b.name.split("_").last
end[0..(0 - keep - 1)]
end
end
end
2 changes: 1 addition & 1 deletion lib/services.rb
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
tag
end

S.register(:today) { Date.today.strftime "%Y-%m-%d" }
S.register(:today) { Time.now.strftime "%Y-%m-%d-%H-%M-%S" }

# Solr stuff

Expand Down
14 changes: 10 additions & 4 deletions spec/authority_browse/solr_spec.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
RSpec.describe AuthorityBrowse::Solr do
context "#list_old_collections" do
it "returns authority_browse collections older than the newest three" do
list = [
before(:each) do
@list = [
"something_11d2069_2023-11-16",
"something_11d2069_2023-11-15",
"something_11d2069_2023-11-14",
Expand All @@ -11,11 +11,17 @@
"authority_browse_11d2069_2023-11-13",
"authority_browse_11d2069_2023-11-15",
"authority_browse_11d2069_2023-11-14"
]
expect(described_class.list_old_collections(list)).to eq([
].map { |x| instance_double(SolrCloud::Collection, name: x) }
end
it "returns authority_browse collections older than the newest three" do
expect(described_class.list_old_collections(list: @list).map { |x| x.name }).to eq([
"authority_browse_1.0.1_2023-11-13",
"authority_browse_11d2069_2023-11-13"
])
end

it "returns old collections with a custom keep_at_least" do
expect(described_class.list_old_collections(list: @list, keep: 4).map { |x| x.name }).to eq(["authority_browse_1.0.1_2023-11-13"])
end
end
end