From 6f748905c7a4a70f06fa32b30ed1f99a02e57cec Mon Sep 17 00:00:00 2001 From: Luigi Asprino Date: Tue, 29 Oct 2024 14:59:37 +0100 Subject: [PATCH] #505 --- README.md | 12 +- .../io/github/sparqlanything/cli/CLI.java | 4 +- .../sparqlanything/cli/SPARQLAnything.java | 142 +++++++++--------- .../io/github/sparqlanything/cli/CLITest.java | 11 ++ .../sparqlanything/cli/LoadSubdirsTest.java | 4 +- 5 files changed, 95 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 6c19ea23..7fc2fa80 100644 --- a/README.md +++ b/README.md @@ -568,12 +568,12 @@ usage: java -jar sparql.anything- -q query [-f ] [-v and not using blank nodes) -e,--explain OPTIONAL - Explain query execution - -l,--load OPTIONAL - The path to one RDF - file or a folder including a set - of files to be loaded. When - present, the data is loaded in - memory and the query executed - against it. + -l,--load OPTIONAL - The path or the URL to + one RDF file or a filepath to a + folder including a set of files + to be loaded. When present, the + data is loaded in memory and the + query executed against it. -f,--format OPTIONAL - Format of the output file. Supported values: JSON, XML, CSV, TEXT, TTL, NT, NQ. diff --git a/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/CLI.java b/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/CLI.java index 7941e279..b9354990 100644 --- a/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/CLI.java +++ b/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/CLI.java @@ -116,8 +116,8 @@ void init(){ options.addOption(Option.builder(EXPLAIN).argName("explain").hasArg(false) .desc("OPTIONAL - Explain query execution").longOpt(EXPLAIN_LONG).build()); - options.addOption(Option.builder(LOAD).argName("load").hasArg().desc( - "OPTIONAL - The path to one RDF file or a folder including a set of files to be loaded. When present, the data is loaded in memory and the query executed against it.") + options.addOption(Option.builder(LOAD).argName("URL or filepath").hasArg().desc( + "OPTIONAL - The path or the URL to one RDF file or a filepath to a folder including a set of files to be loaded. When present, the data is loaded in memory and the query executed against it.") .longOpt(LOAD_LONG).build()); options.addOption(Option.builder(FORMAT).argName("string").hasArg().desc( diff --git a/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/SPARQLAnything.java b/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/SPARQLAnything.java index ef1e391f..d7abb944 100644 --- a/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/SPARQLAnything.java +++ b/sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/SPARQLAnything.java @@ -50,6 +50,7 @@ import java.io.*; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -463,75 +464,8 @@ public static void main(String[] args) throws Exception { if(logger.isTraceEnabled()) { logger.trace("[time] After init: {}", System.currentTimeMillis() - duration); } - Dataset kb = null; - String load = cli.getLoad(); - if (load != null) { - logger.info("Loading data from: {}", load); - if(logger.isTraceEnabled()) { - logger.trace("[time] Before load: {}", System.currentTimeMillis() - duration); - } - // XXX Check if load is a URI first - File loadSource; - try{ - loadSource = new File(new URL(load).toURI()); - }catch(MalformedURLException e){ - loadSource = new File(load); - } - if (loadSource.isDirectory()) { - - logger.info("Loading files from directory: {}", loadSource); - // If directory, load all files - List list = new ArrayList<>(); - Collection files = FileUtils.listFiles(loadSource, null, true); - for (File f : files) { - logger.info("Adding file to be loaded: {}", f); - list.add(f); - } - kb = DatasetFactory.createGeneral(); - for (File f : list) { - try { - Model m = ModelFactory.createDefaultModel(); - // read into the model. - m.read(f.getAbsolutePath()); - kb.addNamedModel(f.toURI().toString(), m); - } catch (Exception e) { - logger.error("An error occurred while loading {}", f); - logger.error(" - Problem was: {}", e.getMessage()); - if(logger.isDebugEnabled()){ - logger.error("",e); - } - } - } - logger.info("Loaded {} triples", kb.asDatasetGraph().getUnionGraph().size()); - } else if (loadSource.isFile()) { - // If it is a file, load it - logger.info("Load file: {}", loadSource); - Path base = Paths.get("."); - try{ - Path p = loadSource.toPath(); - if(!p.isAbsolute()){ - p = base.relativize(loadSource.toPath()); - } - kb = DatasetFactory.create(p.toFile().toURI().toString()); - } catch (Exception e) { - logger.error("An error occurred while loading {}", loadSource); - logger.error(" - Problem was: ", e); - } - } else { - if(!loadSource.exists()){ - logger.error("Option 'load' failed (resource does not exist): {}", loadSource); - }else { - logger.error("Option 'load' failed (not a file or directory): {}", loadSource); - } - return; - } - if(logger.isTraceEnabled()) { - logger.trace("[time] After load: {}", System.currentTimeMillis() - duration); - } - } else { - kb = DatasetFactory.createGeneral(); - } + Dataset kb = createDataset(cli.getLoad()); String outputFileName = cli.getOutputFile(); String outputPattern = cli.getOutputPattern(); @@ -598,6 +532,78 @@ public static void main(String[] args) throws Exception { } } + private static Dataset createDataset(String load) { + Dataset kb = DatasetFactory.createGeneral(); + if (load != null) { + + logger.info("Loading data from: {}", load); + if (logger.isTraceEnabled()) { + logger.trace("[time] Before load: {}", System.currentTimeMillis() - duration); + } + // XXX Check if load is a URI first + File loadSource; + try { + loadSource = new File(new URL(load).toURI()); + } catch (MalformedURLException | URISyntaxException e) { + loadSource = new File(load); + } catch (IllegalArgumentException e) { + Model m = ModelFactory.createDefaultModel(); + RDFDataMgr.read(m, load); + kb.addNamedModel(load, m); + return kb; + } + if (loadSource.isDirectory()) { + + logger.info("Loading files from directory: {}", loadSource); + // If directory, load all files + Collection files = FileUtils.listFiles(loadSource, null, true); + for (File f : files) { + logger.info("Adding file to be loaded: {}", f); + try { + Model m = ModelFactory.createDefaultModel(); + // read into the model. + m.read(f.getAbsolutePath()); + kb.addNamedModel(f.toURI().toString(), m); + } catch (Exception e) { + logger.error("An error occurred while loading {}", f); + logger.error(" - Problem was: {}", e.getMessage()); + if (logger.isDebugEnabled()) { + logger.error("", e); + } + } + } + + logger.info("Loaded {} triples", kb.asDatasetGraph().getUnionGraph().size()); + } else if (loadSource.isFile()) { + // If it is a file, load it + logger.info("Load file: {}", loadSource); + Path base = Paths.get("."); + try { + Path p = loadSource.toPath(); + if (!p.isAbsolute()) { + p = base.relativize(loadSource.toPath()); + } + kb = DatasetFactory.create(p.toFile().toURI().toString()); + } catch (Exception e) { + logger.error("An error occurred while loading {}", loadSource); + logger.error(" - Problem was: ", e); + } + } else { + if (!loadSource.exists()) { + logger.error("Option 'load' failed (resource does not exist): {}", loadSource); + } else { + logger.error("Option 'load' failed (not a file or directory): {}", loadSource); + } + return kb; + } + if (logger.isTraceEnabled()) { + logger.trace("[time] After load: {}", System.currentTimeMillis() - duration); + } + } + + return kb; + } + public static String callMain(String[] args) throws Exception { // Thanks to: https://stackoverflow.com/a/8708357/1035608 // Create a stream to hold the output diff --git a/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/CLITest.java b/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/CLITest.java index 9ce2d432..0d0d5558 100644 --- a/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/CLITest.java +++ b/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/CLITest.java @@ -92,4 +92,15 @@ public void remoteQuery() throws Exception { String queryFile = "https://sparql-anything.cc/examples/CLITestOnFileQuery.sparql"; query(f, queryFile); } + + @Test + public void testRemoteLoad() throws Exception { + Assume.assumeTrue(HTTPHelper.checkHostIsReachable("https://sparql-anything.cc")); + String q = Objects.requireNonNull(getClass().getClassLoader().getResource("count-triples.sparql")).toString(); + String l = "https://sparql-anything.cc/examples/example.ttl"; + String out = SPARQLAnything.callMain(new String[]{ + "-q", q, "-l", l + }); + Assert.assertTrue(out.contains("10")); + } } diff --git a/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/LoadSubdirsTest.java b/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/LoadSubdirsTest.java index 56f6ba61..821f4517 100644 --- a/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/LoadSubdirsTest.java +++ b/sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/LoadSubdirsTest.java @@ -18,6 +18,7 @@ import org.apache.jena.sys.JenaSystem; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +39,7 @@ public void test() throws Exception { String q = Objects.requireNonNull(getClass().getClassLoader().getResource("count-triples.sparql")).toString(); String d = Objects.requireNonNull(getClass().getClassLoader().getResource("./load-subdirs")).toURI().toString(); - SPARQLAnything sa = new SPARQLAnything(); - String out = sa.callMain(new String[]{ + String out = SPARQLAnything.callMain(new String[]{ "-q", q, "-l", d }); Assert.assertTrue(out.contains("51"));