update AgroPortal migration script to handle values xsd:DataType

ontoportal-lirmm · Dec 10, 2023 · a748d01 · a748d01
1 parent f5d4547
commit a748d01
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 31 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 5979402d5138850fb9bdb34edfa350e9af1b5d22
+  revision: 9aa0ccacc9d76bff096218e6d48edc5b0bffd54f
   branch: development
   specs:
     goo (0.0.2)
@@ -26,7 +26,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 7d16b7b1ecc4da8bfd93db2a106737b06d84d6ff
+  revision: 1cfaf4482b7bf8c9001f0ff309f2a0ff06f683b5
   branch: development
   specs:
     ontologies_linked_data (0.0.1)
@@ -74,9 +74,9 @@ GEM
     activesupport (3.2.22.5)
       i18n (~> 0.6, >= 0.6.4)
       multi_json (~> 1.0)
-    addressable (2.8.5)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
-    bcrypt (3.1.19)
+    bcrypt (3.1.20)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
     builder (3.2.4)
@@ -88,8 +88,7 @@ GEM
     debug_inspector (1.1.0)
     declarative (0.0.20)
     docile (1.4.0)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
+    domain_name (0.6.20231109)
     email_spec (2.1.1)
       htmlentities (~> 4.3.3)
       launchy (~> 2.1)
@@ -129,8 +128,11 @@ GEM
       retriable (>= 2.0, < 4.a)
       rexml
       webrick
-    googleauth (1.8.1)
-      faraday (>= 0.17.3, < 3.a)
+    google-cloud-env (2.0.1)
+      faraday (>= 1.0, < 3.a)
+    googleauth (1.9.0)
+      faraday (>= 1.0, < 3.a)
+      google-cloud-env (~> 2.0, >= 2.0.1)
       jwt (>= 1.4, < 3.0)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
@@ -142,8 +144,8 @@ GEM
     httpclient (2.8.3)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json (2.6.3)
-    json_pure (2.6.3)
+    json (2.7.1)
+    json_pure (2.7.1)
     jwt (2.7.1)
     launchy (2.5.2)
       addressable (~> 2.8)
@@ -156,7 +158,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.5.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.1003)
+    mime-types-data (3.2023.1205)
     mini_mime (1.1.5)
     minitest (4.7.5)
     mlanett-redis-lock (0.2.7)
@@ -175,7 +177,7 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.3)
+    public_suffix (5.0.4)
     rack (3.0.8)
     rack-test (2.1.0)
       rack (>= 1.3)
@@ -184,7 +186,7 @@ GEM
       addressable (>= 2.2)
     redis (5.0.8)
       redis-client (>= 0.17.0)
-    redis-client (0.18.0)
+    redis-client (0.19.0)
       connection_pool
     representable (3.2.0)
       declarative (< 0.1.0)
@@ -230,14 +232,12 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uber (0.1.0)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8.2)
     uuid (2.3.9)
       macaddr (~> 1.0)
     webrick (1.8.1)
 
 PLATFORMS
+  x86_64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -269,4 +269,4 @@ DEPENDENCIES
   test-unit-minitest
 
 BUNDLED WITH
-   2.3.15
+   2.4.21
diff --git a/bin/agroportal_model_migration b/bin/agroportal_model_migration
@@ -54,10 +54,8 @@ class AgroPortalMigrator
 
     log_info "Start submission #{sub.id} migration"
     sub.class.model_settings[:attributes][:keyClasses][:enforce].delete(:class)
-    LinkedData::Models::OntologySubmission.agents_attrs.each do |key|
-      sub.class.model_settings[:attributes][key][:enforce].delete(:is_person)
-      sub.class.model_settings[:attributes][key][:enforce].delete(:is_organization)
-    end
+    sub.class.model_settings[:attributes][:mailingList][:enforce].delete(:email)
+
 
     if not_valid?(sub, "submission not valid")
       sub.valid = nil if attribute_error?(sub, :valid)
@@ -75,7 +73,7 @@ class AgroPortalMigrator
       end
 
       if attribute_error?(sub, :designedForOntologyTask)
-        sub.designedForOntologyTask = sub.designedForOntologyTask.map { |x| RDF::URI.new("http://omv.ontoware.org/2005/05/ontology##{x..split(' ').collect(&:capitalize).join}") }
+        sub.designedForOntologyTask = sub.designedForOntologyTask.map { |x| RDF::URI.new(x.to_s) }
       end
 
       if attribute_error?(sub, :modificationDate, :superior_equal_to_creationDate)
@@ -84,13 +82,15 @@ class AgroPortalMigrator
 
       if attribute_error?(sub, :naturalLanguage, :lexvo_language)
         sub.naturalLanguage = Array(sub.naturalLanguage).map do |lang|
-          next lang if lang.to_s.start_with?('http://lexvo.org/id/iso')
-          RDF::URI.new("http://lexvo.org/id/iso639-1/#{lang}")
+          RDF::URI.new("http://lexvo.org/id/iso639-1/#{lang.split('/').last}")
         end
       end
 
       if attribute_error?(sub, :isOfType, :uri)
-        sub.isOfType = RDF::URI.new("http://omv.ontoware.org/2005/05/ontology##{sub.isOfType.to_s.capitalize}")
+        unless sub.isOfType.nil?
+          is_of_type = RDF::URI.new(sub.isOfType)
+          sub.isOfType = is_of_type.valid? ?  is_of_type : RDF::URI.new("http://omv.ontoware.org/2005/05/ontology##{sub.isOfType.split('/').last.capitalize}")
+        end
       end
 
       if attribute_error?(sub, :description)
@@ -99,9 +99,13 @@ class AgroPortalMigrator
 
 
     end
-
-
+    # Delete old string attributes that will be converted to Agents
+    agent_values = {}
     LinkedData::Models::OntologySubmission.agents_attrs.each do |key|
+      sub.class.model_settings[:attributes][key][:enforce].delete(:is_person)
+      is_organization = sub.class.model_settings[:attributes][key][:enforce].delete(:is_organization)
+      sub.class.model_settings[:attributes][key][:enforce].delete(:Agent)
+
       values = sub.send(key)
       is_array = values.is_a?(Array)
       next if values.nil? || values.empty?
@@ -113,7 +117,7 @@ class AgroPortalMigrator
         string_values = string_values.to_s.split(',').map do |value|
           creator = LinkedData::Models::Agent.where(name: value).first
           unless creator
-            creator = LinkedData::Models::Agent.new(name: value, agentType: 'person', creator: admin_user)
+            creator = LinkedData::Models::Agent.new(name: value, agentType: is_organization ? 'organization' : 'person', creator: admin_user)
             stop_to_fix('creator not valid') unless creator.valid?
             creator.save
           end
@@ -123,9 +127,16 @@ class AgroPortalMigrator
         string_values
       end.flatten.compact
 
-      sub.send("#{key}=", is_array ? values : values.first)
+      sub.send("#{key}=", is_array ? [] : nil)
+      values.each do |val|
+        Goo.sparql_update_client.delete_data([sub.id, LinkedData::Models::OntologySubmission.attribute_uri(attr), RDF::Literal.new(val)], graph: sub.graph)
+        Goo.sparql_update_client.delete_data([sub.id, LinkedData::Models::OntologySubmission.attribute_uri(attr), RDF::Literal.new(val, datatype: RDF::XSD.string)], graph: sub.graph)
+      end
+      agent_values[key] = is_array ? values : values.first
     end
 
+    # Delete old string attributes that will be converted to URI
+    uri_values = {}
     sub.errors.each do |key, errors|
       next unless errors.keys.include?(:uri)
 
@@ -139,11 +150,36 @@ class AgroPortalMigrator
 
       sub.class.model_settings[:attributes][key.to_sym][:enforce].delete(:uri)
       sub.send("#{key}=", is_array ? [] : nil)
+      uri_values[key]=  is_array ? values : values.first
+    end
+
+
+    sub.class.model_settings[:attributes][:URI][:enforce].delete(:existence)
+    sub.save rescue stop_to_fix('not valid submission')
+
+    # Save converted strings to URIs
+    begin
+      uri_values.each do |k, v|
+        sub.class.model_settings[:attributes][k.to_sym][:enforce].push(:uri)
+        sub.send("#{k}=",v)
+      end
       sub.save rescue stop_to_fix('not valid submission')
-      sub.class.model_settings[:attributes][key.to_sym][:enforce].push(:uri)
-      sub.send("#{key}=", is_array ? values : values.first)
+    rescue
+      stop_to_fix('URI values migration failed')
     end
 
+    # Save converted strings to Agents
+    begin
+      agent_values.each do |k, v|
+        sub.class.model_settings[:attributes][k.to_sym][:enforce].push(:Agent)
+        sub.send("#{k}=",v)
+      end
+      sub.save rescue stop_to_fix('not valid submission')
+    rescue
+      stop_to_fix('Agent values migration failed')
+    end
+
+
     if sub.valid?
       sub.save rescue stop_to_fix('not valid submission')
       log_info ">> #{sub.id} migrated successfully"

diff --git a/bin/ncbo_ontology_import b/bin/ncbo_ontology_import
@@ -79,6 +79,21 @@ def create_ontology(ont_info)
     new_ontology.viewingRestriction = 'private'
     new_ontology.acl = [USER]
   end
+
+  groups = []
+  has_domains = []
+  Array(ont_info['group']).each do |group|
+    group = LinkedData::Models::Group.find(group.split("/").last).first
+    groups << group if group
+  end
+
+  Array(ont_info['hasDomain']).each do |category|
+    category = LinkedData::Models::Category.find(category.split("/").last).first
+    has_domains << category if category
+  end
+
+  new_ontology.groups = groups
+  new_ontology.hasDomain = has_domains
   new_ontology
 end