Skip to content

Commit

Permalink
Merge pull request #213 from kbss-cvut/development
Browse files Browse the repository at this point in the history 
2.16.0 Release
  • Loading branch information
@ledsoft
ledsoft authored Jan 21, 2023
2 parents 754815c + 3c19e4b commit 2c852f1
Show file tree
Hide file tree
Showing 43 changed files with 2,061 additions and 577 deletions.
11 changes: 9 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</parent>

<artifactId>termit</artifactId>
<version>2.15.0</version>
<version>2.16.0</version>
<name>TermIt</name>
<description>Terminology manager based on Semantic Web technologies.</description>
<packaging>${packaging}</packaging>
Expand Down Expand Up @@ -58,7 +58,7 @@
<dependency>
<groupId>com.github.sgov</groupId>
<artifactId>sgov-validator</artifactId>
<version>1.6.5</version>
<version>1.6.7</version>
</dependency>

<!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
Expand Down Expand Up @@ -296,6 +296,13 @@
<version>5.2.2</version>
</dependency>

<!-- I18n - language tags and languages -->
<dependency>
<groupId>com.neovisionaries</groupId>
<artifactId>nv-i18n</artifactId>
<version>1.29</version>
</dependency>

<!-- Apache Velocity for templating -->
<dependency>
<groupId>org.apache.velocity</groupId>
Expand Down
22 changes: 17 additions & 5 deletions src/main/java/cz/cvut/kbss/termit/dto/PrefixDeclaration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@

import java.util.Objects;

public class PrefixDeclaration {
public class PrefixDeclaration implements Comparable<PrefixDeclaration> {

/**
* Empty prefix declaration, i.e. no prefix is declared.
*/
public static final PrefixDeclaration EMPTY_PREFIX = new PrefixDeclaration(null, null);

/**
* Prefix and local name separator.
Expand All @@ -13,10 +18,6 @@ public class PrefixDeclaration {

private final String namespace;

public PrefixDeclaration() {
this(null, null);
}

public PrefixDeclaration(String prefix, String namespace) {
this.prefix = prefix;
this.namespace = namespace;
Expand Down Expand Up @@ -46,4 +47,15 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(prefix, namespace);
}

@Override
public int compareTo(PrefixDeclaration prefixDeclaration) {
assert prefixDeclaration != null;
if (prefix == null) {
return 1;
} else if (prefixDeclaration.prefix == null) {
return -1;
}
return prefix.compareTo(prefixDeclaration.prefix);
}
}
2 changes: 1 addition & 1 deletion src/main/java/cz/cvut/kbss/termit/persistence/dao/VocabularyDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,7 @@ public PrefixDeclaration resolvePrefix(URI vocabularyUri) {
cz.cvut.kbss.termit.util.Vocabulary.s_p_preferredNamespaceUri))
.getResultList();
if (result.size() == 0) {
return new PrefixDeclaration();
return PrefixDeclaration.EMPTY_PREFIX;
}
assert result.get(0) instanceof Object[];
return new PrefixDeclaration(((Object[]) result.get(0))[0].toString(),
Expand Down
136 changes: 97 additions & 39 deletions src/main/java/cz/cvut/kbss/termit/persistence/validation/Validator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,22 @@
import cz.cvut.kbss.jopa.model.MultilingualString;
import cz.cvut.kbss.termit.exception.TermItException;
import cz.cvut.kbss.termit.model.validation.ValidationResult;
import cz.cvut.kbss.termit.util.Configuration;
import cz.cvut.kbss.termit.util.Utils;
import org.apache.jena.rdf.model.Literal;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.util.FileUtils;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.rio.turtle.TurtleWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;

Expand All @@ -32,65 +35,102 @@
import java.util.stream.Collectors;

@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class Validator implements VocabularyContentValidator {

private static final Logger LOG = LoggerFactory.getLogger(Validator.class);

private final org.eclipse.rdf4j.repository.Repository repository;
private final ValueFactory vf;
/**
* TermIt overrides some glossary validation rules predefined by SGoV.
* <p>
* The main reason for overriding is missing internationalization of the built-in rules.
*/
private static final Set<String> GLOSSARY_RULES_TO_OVERRIDE = Set.of("g2.ttl", "g4.ttl");

/**
* Model validation rules to add.
* <p>
* TermIt does not support all the modeling features validated by SGoV validator. So we use only those we support.
*/
private static final Set<String> MODEL_RULES_TO_ADD = Set.of("m1.ttl", "m2.ttl");

private final EntityManager em;

private com.github.sgov.server.Validator validator;
private Model validationModel;

@Autowired
public Validator(EntityManager em) {
this.repository = em.unwrap(org.eclipse.rdf4j.repository.Repository.class);
vf = repository.getValueFactory();
public Validator(EntityManager em, Configuration config) {
this.em = em;
initValidator(config.getPersistence().getLanguage());
}

private Model getModelFromRdf4jRepository(final Collection<URI> vocabularyIris)
throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);
final RepositoryConnection c = repository.getConnection();
final List<IRI> iris = new ArrayList<>();
vocabularyIris.forEach(i -> iris.add(vf.createIRI(i.toString())));
c.export(new TurtleWriter(writer), iris.toArray(new IRI[]{}));
writer.close();
final byte[] savedData = baos.toByteArray();
final ByteArrayInputStream bais = new ByteArrayInputStream(savedData);
org.apache.jena.rdf.model.Model model = ModelFactory.createDefaultModel();
model.read(bais, null, "TURTLE");
return model;
/**
* Initializes the validator.
* <p>
* Note that the validator can be reused as long as the language stays the same. If TermIt starts supporting
* selection of language per vocabulary, this initialization will have to change.
*
* @param language Primary language of the instance, used to parameterize validation rules
*/
private void initValidator(String language) {
try {
this.validator = new com.github.sgov.server.Validator();
this.validationModel = initValidationModel(validator, language);
} catch (IOException e) {
throw new TermItException("Unable to initialize validator.", e);
}
}

@Transactional(readOnly = true)
@Override
public List<ValidationResult> validate(final Collection<URI> vocabularyIris) {
LOG.debug("Validating {}", vocabularyIris);
final com.github.sgov.server.Validator validator = new com.github.sgov.server.Validator();
private Model initValidationModel(com.github.sgov.server.Validator validator, String language) throws IOException {
final Set<URL> rules = new HashSet<>();
rules.addAll(validator.getGlossaryRules());
rules.addAll(validator.getGlossaryRules().stream()
.filter(r -> GLOSSARY_RULES_TO_OVERRIDE.stream().noneMatch(s -> r.toString().contains(s)))
.collect(Collectors.toSet()));
rules.addAll(
// Currently, only using content rules, not OntoUml, as TermIt does not support adding OntoUml rules
validator.getModelRules().stream().filter(r ->
r.toString().contains("m1.ttl") || r.toString().contains("m2.ttl"))
.collect(Collectors.toList())
validator.getModelRules().stream()
.filter(r -> MODEL_RULES_TO_ADD.stream().anyMatch(s -> r.toString().contains(s)))
.collect(Collectors.toList())
);
final Model validationModel = com.github.sgov.server.Validator.getRulesModel(rules);
loadOverrideRules(validationModel, language);
return validationModel;
}

private void loadOverrideRules(Model validationModel, String language) throws IOException {
final ClassLoader classLoader = Validator.class.getClassLoader();
final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(classLoader);

Resource[] resources = resolver.getResources("classpath:/validation/*.ttl");
for (Resource r : resources) {
String rule = Utils.loadClasspathResource("validation/" + r.getFilename());
rule = rule.replace("$lang", language);
validationModel.read(new ByteArrayInputStream(rule.getBytes(StandardCharsets.UTF_8)), null,
FileUtils.langTurtle);
}
}

@Transactional(readOnly = true)
@Override
public List<ValidationResult> validate(final Collection<URI> vocabularyIris) {
LOG.debug("Validating {}", vocabularyIris);
try {
final Model model = getModelFromRdf4jRepository(vocabularyIris);
org.topbraid.shacl.validation.ValidationReport report = validator.validate(model, rules);
final Model dataModel = getModelFromRdf4jRepository(vocabularyIris);
org.topbraid.shacl.validation.ValidationReport report = validator.validate(dataModel, validationModel);
LOG.debug("Done.");
return report.results().stream()
.sorted(new ValidationResultSeverityComparator()).map(result -> {
.sorted(new ValidationResultSeverityComparator()).map(result -> {
final URI termUri = URI.create(result.getFocusNode().toString());
final URI severity = URI.create(result.getSeverity().getURI());
final URI errorUri = result.getSourceShape().isURIResource() ?
URI.create(result.getSourceShape().getURI()) : null;
URI.create(result.getSourceShape().getURI()) : null;
final URI resultPath = result.getPath() != null && result.getPath().isURIResource() ?
URI.create(result.getPath().getURI()) : null;
URI.create(result.getPath().getURI()) : null;
final MultilingualString messages = new MultilingualString(result.getMessages().stream()
.map(RDFNode::asLiteral)
.collect(Collectors.toMap(Literal::getLanguage, Literal::getLexicalForm)));
.map(RDFNode::asLiteral)
.collect(Collectors.toMap(
Literal::getLanguage,
Literal::getLexicalForm)));

return new ValidationResult()
.setTermUri(termUri)
Expand All @@ -103,4 +143,22 @@ public List<ValidationResult> validate(final Collection<URI> vocabularyIris) {
throw new TermItException("Validation of vocabularies " + vocabularyIris + " failed.", e);
}
}

private Model getModelFromRdf4jRepository(final Collection<URI> vocabularyIris)
throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8);
final Repository repository = em.unwrap(Repository.class);
try (final RepositoryConnection c = repository.getConnection()) {
final List<IRI> iris = new ArrayList<>();
vocabularyIris.forEach(i -> iris.add(repository.getValueFactory().createIRI(i.toString())));
c.export(new TurtleWriter(writer), iris.toArray(new IRI[]{}));
writer.close();
}
final byte[] savedData = baos.toByteArray();
final ByteArrayInputStream bais = new ByteArrayInputStream(savedData);
Model model = ModelFactory.createDefaultModel();
model.read(bais, null, FileUtils.langTurtle);
return model;
}
}
13 changes: 11 additions & 2 deletions src/main/java/cz/cvut/kbss/termit/rest/ResourceController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import cz.cvut.kbss.termit.model.changetracking.AbstractChangeRecord;
import cz.cvut.kbss.termit.model.resource.File;
import cz.cvut.kbss.termit.model.resource.Resource;
import cz.cvut.kbss.termit.rest.util.RestUtils;
import cz.cvut.kbss.termit.security.SecurityConstants;
import cz.cvut.kbss.termit.service.IdentifierResolver;
import cz.cvut.kbss.termit.service.business.ResourceService;
Expand All @@ -42,6 +43,7 @@

import java.io.IOException;
import java.net.URI;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.Set;
Expand Down Expand Up @@ -84,10 +86,17 @@ public void updateResource(@PathVariable String normalizedName,
public ResponseEntity<org.springframework.core.io.Resource> getContent(
@PathVariable String normalizedName,
@RequestParam(name = QueryParams.NAMESPACE, required = false) Optional<String> namespace,
@RequestParam(name = "attachment", required = false) boolean asAttachment) {
@RequestParam(name = "attachment", required = false) boolean asAttachment,
@RequestParam(name = "at", required = false) Optional<String> at) {
final Resource resource = getResource(normalizedName, namespace);
try {
final TypeAwareResource content = resourceService.getContent(resource);
final TypeAwareResource content;
if (at.isPresent()) {
final Instant timestamp = RestUtils.parseTimestamp(at.get());
content = resourceService.getContent(resource, timestamp);
} else {
content = resourceService.getContent(resource);
}
final ResponseEntity.BodyBuilder builder = ResponseEntity.ok()
.contentLength(content.contentLength())
.contentType(MediaType.parseMediaType(
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/cz/cvut/kbss/termit/rest/VocabularyController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ public void runTextAnalysisOnAllTerms(@PathVariable String vocabularyIdFragment,
*/
@GetMapping(value = "/text-analysis")
@ResponseStatus(HttpStatus.ACCEPTED)
@PreAuthorize("hasRole('" + SecurityConstants.ROLE_FULL_USER + "')")
@PreAuthorize("hasRole('" + SecurityConstants.ROLE_ADMIN + "')")
public void runTextAnalysisOnAllVocabularies() {
vocabularyService.runTextAnalysisOnAllVocabularies();
}
Expand Down
Loading

0 comments on commit 2c852f1

Please sign in to comment.