updating libraries

.classpath

@@ -19,26 +19,17 @@
 	<classpathentry kind="lib" path="module/MOD-INF/lib/htmlparser-1.3.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/jackson-core-asl-1.9.5.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/jackson-mapper-asl-1.9.5.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-arq-2.9.0-incubating.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-core-2.7.0-incubating.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-iri-0.9.0-incubating.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-larq-1.0.0-incubating.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
 		<attributes>
 			<attribute name="owner.project.facets" value="java"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/lucene-core-3.6.0.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/commons-logging-1.1.1.jar"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/OpenRefine"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-core-0.9.0.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-mime4j-core-0.7.2.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-mime4j-dom-0.7.2.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/tika-core-1.4.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/tika-parsers-1.4.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-api-0.9.0.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-encoding-0.9.0.jar"/>
-	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-mime-0.9.0.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/sesame-http-client-2.7.5.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/sesame-http-protocol-2.7.5.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/sesame-model-2.7.5.jar"/>
@@ -65,5 +56,26 @@
 	<classpathentry kind="lib" path="module/MOD-INF/lib/sesame-sail-inferencer-2.7.5.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/sesame-sail-memory-2.7.5.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/sesame-util-2.7.5.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/commons-codec-1.6.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/commons-io-2.4.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jcl-over-slf4j-1.6.4.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/lucene-core-4.3.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/lucene-analyzers-common-4.3.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/lucene-queryparser-4.3.1.jar"/>
+	<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/poi-3.8-20120326.jar"/>
+	<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/poi-ooxml-3.8-20120326.jar"/>
+	<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/poi-ooxml-schemas-3.8-20120326.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-tdb-1.0.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-api-0.8.0.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-core-0.8.0.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-encoding-0.8.0.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/apache-any23-mime-0.8.0.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/slf4j-api-1.7.5.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/slf4j-log4j12-1.7.5.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-text-1.0.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-arq-2.11.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-core-2.11.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-iri-1.0.1.jar"/>
+	<classpathentry kind="lib" path="module/MOD-INF/lib/jena-sdb-1.4.1.jar"/>
 	<classpathentry kind="output" path="module/MOD-INF/classes"/>
 </classpath>

module/scripts/rdf-schema-alignment.js

@@ -150,13 +150,13 @@ RdfSchemaAlignmentDialog.prototype._renderBody = function(body) {
     var self = this;
 
     $("#rdf-schema-alignment-tabs").tabs({
-    	activate:function(evt,tabs){
-    		if(tabs.newPanel.index()===2){
+    	select:function(evt,ui){
+    		if(ui.index===1){
     			self._previewRdf();
     		}
     	}
     });
-   // $("#rdf-schema-alignment-tabs-preview").css("display", "");
+    $("#rdf-schema-alignment-tabs-preview").css("display", "");
    // $("#rdf-schema-alignment-tabs-vocabulary-manager").css("display", "");
 
     this._canvas = $(".schema-alignment-dialog-canvas");

src/org/deri/grefine/rdf/vocab/IVocabularySearcher.java

@@ -5,7 +5,6 @@
 import java.util.Set;
 
 import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.queryParser.ParseException;
 import org.openrdf.repository.Repository;
 
 
@@ -32,9 +31,9 @@ public interface IVocabularySearcher {
 	public void importAndIndexVocabulary(String name, String uri, String fetchUrl,String projectId,VocabularyImporter importer) throws VocabularyImportException, VocabularyIndexException, PrefixExistException, CorruptIndexException, IOException;
 	public void importAndIndexVocabulary(String name, String uri, Repository repository, String projectId,VocabularyImporter importer) throws VocabularyImportException, VocabularyIndexException, PrefixExistException, CorruptIndexException, IOException;
 
-	public List<SearchResultItem> searchClasses(String str, String projectId) throws ParseException,IOException;
+	public List<SearchResultItem> searchClasses(String str, String projectId) throws IOException;
 
-	public List<SearchResultItem> searchProperties(String str, String projectId) throws ParseException,IOException;
+	public List<SearchResultItem> searchProperties(String str, String projectId) throws IOException;
 
 	public void deleteTermsOfVocabs(Set<Vocabulary> toRemove,String projectId) throws CorruptIndexException, IOException;
 	public void deleteTermsOfVocab(String vocabName, String projectId) throws CorruptIndexException, IOException;

src/org/deri/grefine/rdf/vocab/imp/VocabularySearcher.java

@@ -10,18 +10,18 @@
 import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.core.SimpleAnalyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
@@ -58,17 +58,18 @@ public class VocabularySearcher implements IVocabularySearcher {
 
 	private IndexWriter writer;
 	private IndexSearcher searcher;
+	private IndexReader r;
 
 	private Directory _directory;
 
 	public VocabularySearcher(File dir) throws IOException {
                 _directory = new SimpleFSDirectory(new File(dir, "luceneIndex"));
-                Analyzer a = new SimpleAnalyzer(Version.LUCENE_36);
-                IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36,a);                
+                Analyzer a = new SimpleAnalyzer(Version.LUCENE_43);
+                IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43,a);                
 
                 writer = new IndexWriter(_directory,conf);
                 writer.commit();
-                IndexReader r = IndexReader.open(_directory);
+                r = DirectoryReader.open(_directory);
                 searcher = new IndexSearcher(r);
         }
 
@@ -98,15 +99,15 @@ public void importAndIndexVocabulary(String name, String uri, Repository reposit
 
 	@Override
 	public List<SearchResultItem> searchClasses(String str, String projectId)
-			throws ParseException, IOException {
+			throws IOException {
 		Query query = prepareQuery(str, CLASS_TYPE, projectId);
 		TopDocs docs = searcher.search(query, getMaxDoc());		
 		return prepareSearchResults(docs);
 	}
 
 	@Override
 	public List<SearchResultItem> searchProperties(String str, String projectId)
-			throws ParseException, IOException {
+			throws IOException {
 		Query query = prepareQuery(str, PROPERTY_TYPE, projectId);
 		TopDocs docs = searcher.search(query, getMaxDoc());
 		return prepareSearchResults(docs);
@@ -136,8 +137,9 @@ public void update() throws CorruptIndexException, IOException {
 		writer.commit();
 		// TODO this shouldn't be required but it is not working without it...
 		// check
-		searcher.close();
-		searcher = new IndexSearcher(IndexReader.open(_directory));
+		r.close();
+		r = IndexReader.open(_directory);
+		searcher = new IndexSearcher(r);
 	}
 
 	@Override
@@ -220,7 +222,7 @@ private void indexRdfNode(RDFNode node, String type, String projectId)
 	}
 
 	private Query prepareQuery(String s, String type, String projectId)
-			throws ParseException, IOException {
+			throws IOException {
 		BooleanQuery q1 = new BooleanQuery();
 		// q1.add(new TermQuery(new
 		// Term("projectId",GLOBAL_VOCABULARY_PLACE_HOLDER)), Occur.SHOULD);
@@ -357,7 +359,7 @@ private void addDocumentsToProject(TopDocs docs,String projectId) throws Corrupt
 			Document newdoc = new Document();
 			Iterator fieldsIter = doc.getFields().iterator();
 			while(fieldsIter.hasNext()){
-				newdoc.add((Fieldable)fieldsIter.next());
+				newdoc.add((IndexableField)fieldsIter.next());
 			}
 			newdoc.removeField("projectId");
 			newdoc.add(new Field("projectId",projectId,Field.Store.YES,Field.Index.NOT_ANALYZED));
@@ -407,6 +409,6 @@ private void deletePrefixesOfProjectId(String projectId, Set<String> toDelete) t
 	}
 
 	private int getMaxDoc() throws IOException {
-		return searcher.maxDoc() > 0 ? searcher.maxDoc() : 100000;
+		return r.maxDoc() > 0 ? r.maxDoc() : 100000;
 	}
 }

src/org/deri/grefine/reconcile/rdf/executors/DumpQueryExecutor.java

@@ -4,21 +4,24 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 
-import org.apache.jena.larq.IndexBuilderString;
-import org.apache.jena.larq.IndexLARQ;
-import org.apache.jena.larq.LARQ;
+import org.apache.jena.query.text.EntityDefinition;
+import org.apache.jena.query.text.TextDatasetFactory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
+import com.hp.hpl.jena.query.Dataset;
+import com.hp.hpl.jena.query.DatasetFactory;
 import com.hp.hpl.jena.query.Query;
 import com.hp.hpl.jena.query.QueryExecution;
 import com.hp.hpl.jena.query.QueryExecutionFactory;
 import com.hp.hpl.jena.query.QueryFactory;
+import com.hp.hpl.jena.query.ReadWrite;
 import com.hp.hpl.jena.query.ResultSet;
 import com.hp.hpl.jena.query.Syntax;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelFactory;
-import com.hp.hpl.jena.rdf.model.Property;
 
 /**
  * @author fadmaa
@@ -29,8 +32,7 @@
  */
 public class DumpQueryExecutor implements QueryExecutor {
 
-	private Model model;
-	private IndexLARQ index;
+	private Dataset index;
 	private boolean loaded = false;
 	//property used for index/search (only if one property is used)
 	private String propertyUri;
@@ -63,28 +65,28 @@ public DumpQueryExecutor(Model m){
 
 	public DumpQueryExecutor(Model m, String propertyUri, boolean ngramIndex,int minGram, int maxGram){
 		loaded = true;
-		this.model = m;
 		this.propertyUri = propertyUri;
-		IndexBuilderString larqBuilder;
-		if(propertyUri != null){
-			Property p = model.getProperty(propertyUri);
-			larqBuilder= new IndexBuilderString(p) ;
-		}else{
-			larqBuilder= new IndexBuilderString() ;
-		}
-		larqBuilder.indexStatements(model.listStatements()) ;
-		larqBuilder.closeWriter() ;
-		this.index = larqBuilder.getIndex() ;
+
+		Dataset ds1 = DatasetFactory.createMem();
+		EntityDefinition entDef = new EntityDefinition("uri", "text",m.getResource(propertyUri)) ;
+
+        // Lucene, in memory.
+        Directory dir =  new RAMDirectory();
+
+        // Join together into a dataset
+        this.index = TextDatasetFactory.createLucene(ds1, dir, entDef) ;
+        this.index.getDefaultModel().add(m);
+        //this.index.commit();
 	}
 
 	@Override
 	public ResultSet sparql(String sparql) {
 		if(!loaded){
 			throw new RuntimeException("Model is not loaded");
 		}
+		//this.index.begin(ReadWrite.READ) ;
 		Query query = QueryFactory.create(sparql, Syntax.syntaxSPARQL_11);
-		QueryExecution qExec = QueryExecutionFactory.create(query, model);
-		LARQ.setDefaultIndex(qExec.getContext(), index);
+		QueryExecution qExec = QueryExecutionFactory.create(query, this.index);
 		ResultSet result =  qExec.execSelect();
 		return result;
 	}
@@ -101,7 +103,8 @@ public void write(JSONWriter writer) throws JSONException {
 	}
 
 	public void dispose(){
-		model = null; //free the memory used for the model
+		this.index.close();
+		this.index = null; //free the memory used for the model
 	}
 
 	public synchronized void initialize(FileInputStream in) {
@@ -110,30 +113,27 @@ public synchronized void initialize(FileInputStream in) {
 		}
 		loaded = true;
 		// -- Read and index all literal strings.
-		IndexBuilderString larqBuilder;
-		model = ModelFactory.createDefaultModel();
+		Model model = ModelFactory.createDefaultModel();
 		model.read(in, null,"TTL");
-		if(propertyUri==null){
-			larqBuilder = new IndexBuilderString() ;
-		}else{
-			Property p = model.getProperty(propertyUri);
-			larqBuilder = new IndexBuilderString(p);
-		}
-
-		larqBuilder.indexStatements(model.listStatements()) ;
-		// -- Finish indexing
-		larqBuilder.closeWriter() ;
+
+		Dataset ds1 = DatasetFactory.createMem();
+		EntityDefinition entDef = new EntityDefinition("uri", "text",model.getResource(propertyUri)) ;
 
-		// -- Create the access index  
-		index = larqBuilder.getIndex() ;
+        // Lucene, in memory.
+        Directory dir =  new RAMDirectory();
+
+        // Join together into a dataset
+        this.index = TextDatasetFactory.createLucene(ds1, dir, entDef) ;
+        this.index.getDefaultModel().add(model);
+        this.index.commit();
 	}
 
 	private static final int DEFAULT_MIN_NGRAM = 3;
 	private static final int DEFAULT_MAX_NGRAM = 3;
 
 	@Override
 	public void save(String serviceId, FileOutputStream out) throws IOException {
-		model.write(out, "TTL");
+		this.index.getDefaultModel().write(out, "TTL");
 		out.close();
 	}
 }

src/org/deri/grefine/reconcile/rdf/factories/LarqSparqlQueryFactory.java

@@ -3,7 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryparser.classic.QueryParser;
 import org.deri.grefine.reconcile.model.ReconciliationRequest;
 import org.deri.grefine.reconcile.model.SearchResultItem;
 import org.deri.grefine.reconcile.model.ReconciliationRequestContext.PropertyContext;
@@ -70,6 +70,7 @@ private String getReconciliationSparqlQuery(String queryTemplate, ImmutableList<
 		//the answers are ordered according to their scores descendingly. thus we need to pick only the *first* request.getLimit() *unique* entity answer
 		int calculatedLimit = Math.max(searchPropertyUris.size(),1) * limit;
 		return queryTemplate.replace("[[QUERY]]", escapedQuery)
+						.replace(labelPlaceHolder, labelFilter)
 						.replace(labelPlaceHolder, labelFilter)
 						.replace("[[TYPE_FILTER]]", typesFilter)
 						.replace("[[CONTEXT_FILTER]]", contextFilter)
@@ -233,13 +234,13 @@ public double getScore() {
 				"ORDER BY DESC(?score1) LIMIT [[LIMIT]]";
 	private static final String SINGLE_LABEL_PROPERTY_RECONCILE_QUERY_TEMPLATE =
 		"PREFIX rdfs:<http://www.w3.org/2000/01/rdf-schema#> " + 
-		"PREFIX pf:<http://jena.hpl.hp.com/ARQ/property#> " +
+		"PREFIX text:<http://jena.apache.org/text#> " +
 		"PREFIX rdf:<http://www.w3.org/1999/02/22-rdf-syntax-ns#> "+  
-		"SELECT ?entity ?label (MAX(?score) AS ?score1) " +
-		"WHERE" +
-		"{" +
-			"?entity <[[LABEL_PROPERTY_URI]]> ?label." +
-			"(?label ?score) pf:textMatch '[[QUERY]]'. " +
+		"SELECT ?entity ?label " +
+		"WHERE " +
+		"{ " +
+			"?entity text:query (<[[LABEL_PROPERTY_URI]]> '[[QUERY]]' [[LIMIT]]) . " +
+			"?entity <[[LABEL_PROPERTY_URI]]> ?label ." +
 			"[[TYPE_FILTER]]" +
 			"[[CONTEXT_FILTER]]" +
 		"}GROUP BY ?entity ?label " +