[#109] Add google docs conversion support

See also
OpenDataServices/cove@2e3eb5d

fts/test_dataload.py

@@ -2,6 +2,7 @@
 from selenium import webdriver
 import os
 import time
+import requests
 
 
 server_url = os.environ['SERVER_URL']
@@ -15,6 +16,18 @@ def browser(request):
     return browser
 
 
+def test_googledoc_input(browser):
+    source_url = 'https://docs.google.com/spreadsheets/d/1lZBVe6TGfrPYtaP4bqP9OZ0WG26vh_jTzOOm1h2Qdc4/edit#gid=0'
+
+    browser.get(server_url + prefix)
+    browser.find_element_by_id('id_source_url').send_keys(source_url)
+    browser.find_element_by_css_selector("#fetchURL > div.form-group > button.btn.btn-primary").click()
+
+    browser.find_element_by_css_selector("button.btn.btn-default.convert").click()
+    ttl_href = browser.find_element_by_link_text('Outputted TTL').get_attribute('href')
+    assert 'Block 15' in requests.get(ttl_href).text
+
+
 def test_humanize_naturaltime(browser):
     '''
       This test loads a file from a url into the resourceprojects interface.
@@ -26,8 +39,6 @@ def test_humanize_naturaltime(browser):
     source_url = 'https://raw.githubusercontent.com/NRGI/resource-projects-etl/ee55c2956d23ebfc7a71cb1994a149d966a3c2a7/fts/fixtures/statoil-4-rows.csv'
 
     browser.get(server_url + prefix)
-    browser.find_element_by_partial_link_text('Link').click()
-    time.sleep(0.5)
     browser.find_element_by_id('id_source_url').send_keys(source_url)
     browser.find_element_by_css_selector("#fetchURL > div.form-group > button.btn.btn-primary").click()
 

modules/cove_resourceprojects.py

@@ -6,6 +6,7 @@
 #from requests.auth import HTTPDigestAuth
 import subprocess
 import os
+import json
 import urllib.parse
 
 
@@ -14,13 +15,27 @@ def fetch(dataset):
 
 
 def convert(dataset):
+    if dataset.supplied_data.is_google_doc():
+        main_source = None
+    else:
+        main_source = dataset.supplied_data.original_file.file.name
+
     tl = TagLifter(
         ontology="ontology/resource-projects-ontology.rdf",
-        source=dataset.supplied_data.original_file.file.name,
+        source=main_source,
         base="http://resourceprojects.org/",
-        source_meta={"author": "TODO", "Source_Type": "official", "Converted": "Today"}
+        source_meta={}
     )
-    tl.build_graph()
+
+    if dataset.supplied_data.is_google_doc():
+        with open(dataset.supplied_data.original_file.file.name) as fp:
+            for entry in json.load(fp)['feed']['entry']:
+                tl.source_meta = {"sheet": entry['title']['$t']}
+                tl.load_data(os.path.join(dataset.supplied_data.upload_dir(), 'googledoc', entry['title']['$t']))
+                tl.build_graph()
+    else:
+        tl.build_graph()
+
     tl.graph.serialize(
         format='turtle',
         destination=os.path.join(dataset.supplied_data.upload_dir(), 'output.ttl')
@@ -86,6 +101,11 @@ def delete_from_virtuoso(dataset, staging):
     ])
 
 
+def outputted_ttl(dataset):
+     return dataset.supplied_data.upload_url() + 'output.ttl'
+
+
+
 PROCESSES = OrderedDict([
     ('fetch', {
         'name': 'Fetched',
@@ -97,7 +117,8 @@ def delete_from_virtuoso(dataset, staging):
     ('convert', {
         'name': 'Converted',
         'action_name': 'Convert',
-        'more_info_name': 'Conversion messages',
+        'more_info_name': 'Outputted TTL',
+        'more_info_link': outputted_ttl,
         'depends': 'fetch',
         'function': convert,
         'main': True

requirements.txt

@@ -1,4 +1,4 @@
--e git+https://github.com/OpenDataServices/cove.git@44177c4a6dcb30ea28d9f963ca7a6994801b6195#egg=cove
+-e git+https://github.com/OpenDataServices/cove.git@2e3eb5da5354799063e7fc16831c7acf2b2ef5a2#egg=cove
 -e git+https://github.com/OpenDataServices/flatten-tool.git@61d8404b444f10384363cde1cad542a0d04af004#egg=flattentool
 -r requirements_taglifter.txt
 gunicorn==19.3.0

requirements_test.txt

@@ -1,5 +1,6 @@
 py==1.4.28
 pytest==2.7.1
+requests==2.7.0
 
 # selenium
 selenium==2.46.0