From d7ee80860a0eab9293af81083a0700d099c50263 Mon Sep 17 00:00:00 2001 From: mdorf Date: Fri, 15 Dec 2023 14:29:03 -0800 Subject: [PATCH 1/5] an improved fix to ncbo/ncbo_ontology_recommender#21 --- lib/ncbo_annotator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ncbo_annotator.rb b/lib/ncbo_annotator.rb index caf2198a..4631b53a 100644 --- a/lib/ncbo_annotator.rb +++ b/lib/ncbo_annotator.rb @@ -725,7 +725,7 @@ def mappings_for_class_ids(class_ids) def hierarchy_query(class_ids) # mdorf, 12/14/2023: AllegroGraph throws a MalformedQuery exception # if an ID is not of the proper URI format - class_ids.select! { |id| id =~ URI::regexp } + class_ids.select! { |id| id =~ /\A#{URI::regexp}\z/ } filter_ids = class_ids.map { |id| "?id = <#{id}>" } .join " || " query = < ?parent . } From 944af97d38d874497f1db28a206a0e9d3b753d17 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 19 Dec 2023 16:41:20 -0800 Subject: [PATCH 2/5] Add AllegroGraph backend to unit test matrix --- .github/workflows/ruby-unit-tests.yml | 8 +- Dockerfile | 3 +- config/config.test.rb | 33 ++++---- docker-compose.yml | 108 ++++++++++++++++++++++---- run-unit-tests.sh | 5 +- 5 files changed, 122 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index f5e84715..c4268d19 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -7,6 +7,10 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + backend: ['ruby', 'ruby-agraph'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend steps: - uses: actions/checkout@v3 - name: copy config file @@ -14,9 +18,11 @@ jobs: - name: Build docker-compose run: docker-compose build - name: Run unit tests + # unit tests are run inside a container + # http://docs.codecov.io/docs/testing-with-docker run: | ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI ruby bundle exec rake test TESTOPTS='-v' + docker-compose run $ci_env -e CI ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: diff --git a/Dockerfile b/Dockerfile index 5521fe1e..5016faa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,8 @@ COPY *.gemspec Gemfile* /srv/ontoportal/ncbo_annotator/ WORKDIR /srv/ontoportal/ncbo_annotator -RUN gem install bundler +# set bundler to v2.4.22 which is the last version supported by ruby 2.7 +RUN gem install bundler -v 2.4.22 ENV BUNDLE_PATH /bundle RUN bundle install COPY . /srv/ontoportal/ncbo_annotator diff --git a/config/config.test.rb b/config/config.test.rb index b915f03d..bfba7b91 100644 --- a/config/config.test.rb +++ b/config/config.test.rb @@ -1,24 +1,31 @@ # frozen_string_literal: true # -# This file is designed for used for unit testing +# configuration for unit testing # -# All the defaults are set in -# https://github.com/ncbo/ontologies_linked_data/blob/master/lib/ontologies_linked_data/config/config.rb -### -GOO_HOST = ENV.include?('GOO_HOST') ? ENV['GOO_HOST'] : 'localhost' -GOO_PORT = ENV.include?('GOO_PORT') ? ENV['GOO_PORT'] : 9000 -REDIS_HOST = ENV.include?('REDIS_HOST') ? ENV['REDIS_HOST'] : 'localhost' -REDIS_PORT = ENV.include?('REDIS_PORT') ? ENV['REDIS_PORT'] : 6379 -SOLR_HOST = ENV.include?('SOLR_HOST') ? ENV['SOLR_HOST'] : 'localhost' -MGREP_HOST = ENV.include?('MGREP_HOST') ? ENV['MGREP_HOST'] : 'localhost' -MGREP_PORT = ENV.include?('MGREP_PORT') ? ENV['MGREP_PORT'] : 55556 +GOO_HOST = ENV.include?('GOO_HOST') ? ENV['GOO_HOST'] : 'localhost' +GOO_PORT = ENV.include?('GOO_PORT') ? ENV['GOO_PORT'] : 9000 +GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store" +GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" +GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/data/" +GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" +REDIS_HOST = ENV.include?('REDIS_HOST') ? ENV['REDIS_HOST'] : 'localhost' +REDIS_PORT = ENV.include?('REDIS_PORT') ? ENV['REDIS_PORT'] : 6379 +MGREP_HOST = ENV.include?('MGREP_HOST') ? ENV['MGREP_HOST'] : 'localhost' +MGREP_PORT = ENV.include?('MGREP_PORT') ? ENV['MGREP_PORT'] : 55556 +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" + LinkedData.config do |config| + config.goo_backend_name = GOO_BACKEND_NAME.to_s config.goo_host = GOO_HOST.to_s config.goo_port = GOO_PORT.to_i - config.search_server_url = "http://#{SOLR_HOST}:8983/solr/term_search_core1".to_s - config.property_search_server_url = "http://#{SOLR_HOST}:8983/solr/prop_search_core1".to_s + config.goo_path_query = GOO_PATH_QUERY.to_s + config.goo_path_data = GOO_PATH_DATA.to_s + config.goo_path_update = GOO_PATH_UPDATE.to_s + config.search_server_url = SOLR_TERM_SEARCH_URL.to_s + config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s end Annotator.config do |config| config.mgrep_host = MGREP_HOST.to_s diff --git a/docker-compose.yml b/docker-compose.yml index 43c9aa0e..3b343882 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,39 +1,89 @@ version: '3.8' -services: - ruby: +x-app: &app build: context: . args: RUBY_VERSION: '2.7' + environment: &env + COVERAGE: 'true' # enable simplecov code coverage + REDIS_HOST: redis-ut + REDIS_PORT: 6379 + MGREP_HOST: mgrep-ut + MGREP_PORT: 55556 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 volumes: - #- bundle:/bundle - .:/srv/ontoportal/ncbo_annotator + stdin_open: true + tty: true + command: "bundle exec rake test TESTOPTS='-v'" + depends_on: &depends_on + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + mgrep-ut: + condition: service_healthy + +services: + ruby: + <<: *app environment: - - BUNDLE_PATH=/bundle - - GOO_BACKEND_NAME=4store - - GOO_PORT=9000 - - GOO_HOST=4store-ut - - REDIS_HOST=redis-ut - - REDIS_PORT=6379 - - SOLR_HOST=solr-ut - - MGREP_HOST=mgrep-ut - - MGREP_PORT=55556 + <<: *env + GOO_BACKEND_NAME: 4store + GOO_PORT: 9000 + GOO_HOST: 4store-ut depends_on: - - solr-ut - - redis-ut - - 4store-ut - - mgrep-ut - command: "bundle exec rake test TESTOPTS='-v'" + <<: *depends_on + 4store-ut: + condition: service_started + profiles: + - 4store + + ruby-agraph: + <<: *app + environment: + <<: *env + GOO_BACKEND_NAME: ag + GOO_PORT: 10035 + GOO_HOST: agraph-ut + GOO_PATH_QUERY: /repositories/bioportal_test + GOO_PATH_DATA: /repositories/bioportal_test/statements + GOO_PATH_UPDATE: /repositories/bioportal_test/statements + depends_on: + <<: *depends_on + agraph-ut: + condition: service_healthy + profiles: + - agraph solr-ut: image: ontoportal/solr-ut:0.0.2 + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] + start_period: 3s + interval: 10s + timeout: 5s + retries: 5 redis-ut: image: redis + healthcheck: + test: redis-cli ping + interval: 10s + timeout: 3s + retries: 10 mgrep-ut: image: ontoportal/mgrep:0.0.2 + platform: linux/amd64 + healthcheck: + test: ["CMD", "nc", "-z", "-v", "localhost", "55556"] + start_period: 3s + interval: 10s + timeout: 5s + retries: 5 4store-ut: image: bde2020/4store @@ -42,6 +92,30 @@ services: bash -c "4s-backend-setup --segments 4 ontoportal_kb && 4s-backend ontoportal_kb && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" + profiles: + - 4store + + agraph-ut: + image: franzinc/agraph:v8.0.0 + platform: linux/amd64 + environment: + - AGRAPH_SUPER_USER=test + - AGRAPH_SUPER_PASSWORD=xyzzy + shm_size: 1g + command: > + bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start + ; agtool repos create bioportal_test + ; agtool users add anonymous + ; agtool users grant anonymous root:bioportal_test:rw + ; tail -f /agraph/data/agraph.log" + healthcheck: + test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"] + start_period: 30s + interval: 10s + timeout: 5s + retries: 5 + profiles: + - agraph volumes: bundle: diff --git a/run-unit-tests.sh b/run-unit-tests.sh index cc239130..476c2b83 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -4,8 +4,7 @@ # add config for unit testing [ -f config/config.rb ] || cp config/config.test.rb config/config.rb docker-compose build -#docker-compose up --exit-code-from unit-test docker-compose run --rm ruby bundle exec rake test TESTOPTS='-v' -#docker-compose down -docker-compose kill +#docker-compose run --rm ruby-agraph bundle exec rake test TESTOPTS='-v' +docker-compose --profile agraph --profile 4store kill From 354b388cf652e55cd534c68c9db0ccdf60bc167a Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 19 Dec 2023 17:08:37 -0800 Subject: [PATCH 3/5] minitest style change for troubleshooting purposes --- test/test_annotator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_annotator.rb b/test/test_annotator.rb index 08a3736d..755c99c1 100644 --- a/test/test_annotator.rb +++ b/test/test_annotator.rb @@ -97,7 +97,7 @@ def test_generate_dictionary_file assert @@redis.exists?(Annotator::Models::NcboAnnotator::MGREP_DICTIONARY_REFRESH_TIMESTAMP) assert @@redis.exists?(Annotator::Models::NcboAnnotator::LAST_MGREP_RESTART_TIMESTAMP) refresh_timestamp = @@redis.get(Annotator::Models::NcboAnnotator::MGREP_DICTIONARY_REFRESH_TIMESTAMP) - assert refresh_timestamp > start_timestamp + assert_operator refresh_timestamp, :>, start_timestamp end def test_mallet_recognizer From 9a037b955b1d6c3c9955250d7499afdd10d4bbd3 Mon Sep 17 00:00:00 2001 From: mdorf Date: Wed, 10 Jan 2024 12:02:24 -0800 Subject: [PATCH 4/5] Gemfile.lock update --- Gemfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8c251895..0dd16913 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 6db93bb3d5095a5fe0d017e572c5a04caa34ebc6 + revision: db2b330fb6c5fd4ea9ee17d5b58ca997f304a340 branch: develop specs: goo (0.0.2) @@ -15,7 +15,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 9487c7f73e68abab097af523d42c1d2e106e614b + revision: e716a6d410883a8e59121e7d09d2c80a6f6ab1fc branch: develop specs: ontologies_linked_data (0.0.1) @@ -63,7 +63,7 @@ GEM cube-ruby (0.0.3) date (3.3.4) docile (1.4.0) - domain_name (0.6.20231109) + domain_name (0.6.20240107) faraday (1.10.3) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) @@ -95,7 +95,7 @@ GEM concurrent-ruby (~> 1.0) json (2.7.1) json_pure (2.7.1) - libxml-ruby (4.1.2) + libxml-ruby (5.0.2) logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) @@ -105,7 +105,7 @@ GEM net-pop net-smtp method_source (1.0.0) - mime-types (3.5.1) + mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2023.1205) mini_mime (1.1.5) @@ -120,7 +120,7 @@ GEM net-protocol net-protocol (0.2.2) timeout - net-smtp (0.4.0) + net-smtp (0.4.0.1) net-protocol netrc (0.11.0) oj (3.16.3) @@ -141,7 +141,7 @@ GEM addressable (>= 2.2) redis (5.0.8) redis-client (>= 0.17.0) - redis-client (0.19.0) + redis-client (0.19.1) connection_pool rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) From f60341e8426adcad447a6fc3adb5b80207c5987a Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 12 Jan 2024 15:07:26 -0800 Subject: [PATCH 5/5] Gemfile.lock update --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0dd16913..412e9f9b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -15,7 +15,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: e716a6d410883a8e59121e7d09d2c80a6f6ab1fc + revision: 4b6b7f42882b9ad815ff5e90c354212aca085fda branch: develop specs: ontologies_linked_data (0.0.1) @@ -113,7 +113,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.3.7) + net-imap (0.4.9.1) date net-protocol net-pop (0.1.2) @@ -192,4 +192,4 @@ DEPENDENCIES sparql-client! BUNDLED WITH - 2.3.15 + 2.4.22