Skip to content
Snippets Groups Projects
Commit feca8ea4 authored by Jesper Zedlitz's avatar Jesper Zedlitz
Browse files

Merge branch '8-dataservice-und-datasetseries-erhalten' into 'master'

Resolve "DataService und DatasetSeries erhalten"

Closes #8

See merge request !5
parents a439a2a2 7a55e1ac
Branches master
No related tags found
1 merge request!5Resolve "DataService und DatasetSeries erhalten"
Pipeline #1256 passed
<?xml version="1.0" encoding="utf-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.2.2</version>
<version>3.4.0</version>
<relativePath />
</parent>
<groupId>de.landsh.opendata</groupId>
<artifactId>dcat-catalog-proxy</artifactId>
<version>1.3.0</version>
<version>1.4.0-SNAPSHOT</version>
<name>dcat-catalog-proxy</name>
<description>DCAT catalog proxy</description>
<properties>
<java.version>17</java.version>
<jena.version>4.7.0</jena.version>
<jena.version>5.2.0</jena.version>
</properties>
<dependencies>
<dependency>
......@@ -80,6 +79,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<artifactId>snakeyaml</artifactId>
<version>1.32</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.18.0</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
......@@ -92,7 +97,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.7</version>
<version>0.8.12</version>
<executions>
<execution>
<id>coverage-initialize</id>
......@@ -157,7 +162,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>7.4.4</version>
<version>11.1.0</version>
<configuration>
<suppressionFiles>
<suppressionFile>owasp-suppressions.xml</suppressionFile>
......@@ -165,6 +170,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<failBuildOnCVSS>8</failBuildOnCVSS>
<assemblyAnalyzerEnabled>false</assemblyAnalyzerEnabled>
<failOnError>true</failOnError>
<nvdApiKey>${env.NVD_API_KEY}</nvdApiKey>
</configuration>
<executions>
<execution>
......@@ -177,7 +183,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>2.14.2</version>
<version>2.18.0</version>
<configuration>
<rulesUri>file:///${project.basedir}/rules.xml</rulesUri>
</configuration>
......@@ -187,14 +193,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
<artifactId>spotbugs-maven-plugin</artifactId>
<version>4.5.3.0</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>2.14.2</version>
<configuration>
<rulesUri>file:///${project.basedir}/rules.xml</rulesUri>
</configuration>
</plugin>
</plugins>
</build>
</project>
......@@ -11,15 +11,5 @@
<ignoreVersion type="regex">(?i).*M(?:-?\d+)?</ignoreVersion>
</ignoreVersions>
<rules>
<!-- 4.7.0 is the last version that work. There is an additional rule for
IRIs in later versions that prevents some of CKAN's responses
to be parsed. -->
<rule groupId="org.apache.jena" comparisonMethod="maven">
<ignoreVersions>
<ignoreVersion type="regex">4.[89].*</ignoreVersion>
<ignoreVersion type="regex">4.1[0-9].*</ignoreVersion>
</ignoreVersions>
</rule>
</rules>
</ruleset>
......@@ -14,6 +14,7 @@ import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import java.io.InputStream;
import java.nio.channels.Selector;
import java.util.*;
/**
......@@ -129,8 +130,9 @@ public class CatalogFilter implements InitializingBean {
final Resource accessURL = distribution.getPropertyResourceValue(DCAT.accessURL);
final Resource downloadURL = distribution.getPropertyResourceValue(DCAT.downloadURL);
final boolean hasAccessService = distribution.hasProperty(DCAT.accessService);
if (downloadURL == null) {
if (downloadURL == null && !hasAccessService) {
distribution.addProperty(DCAT.downloadURL, accessURL);
}
}
......@@ -359,8 +361,7 @@ public class CatalogFilter implements InitializingBean {
void changeStartAndEndDate(Model model) {
LinkedList<Statement> oldStatements = new LinkedList<>();
Selector selectorStart = new SimpleSelector(null, model.createProperty("http://schema.org/startDate"), (Object) null);
StmtIterator it = model.listStatements(selectorStart);
StmtIterator it = model.listStatements(null, model.createProperty("http://schema.org/startDate"), (String) null);
while (it.hasNext()) {
Statement stmt = it.next();
oldStatements.add(stmt);
......@@ -368,8 +369,7 @@ public class CatalogFilter implements InitializingBean {
}
it.close();
Selector selectorEnd = new SimpleSelector(null, model.createProperty("http://schema.org/endDate"), (Object) null);
it = model.listStatements(selectorEnd);
it = model.listStatements(null, model.createProperty("http://schema.org/endDate"), (String) null);
while (it.hasNext()) {
Statement stmt = it.next();
oldStatements.add(stmt);
......
package de.landsh.opendata.catalogproxy;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jena.irix.IRIException;
import org.apache.jena.irix.IRIs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
......@@ -10,8 +15,9 @@ import java.nio.charset.StandardCharsets;
public class FilterInvalidRDF extends InputStream {
private static final String RDF_ABOUT_DOUBLE_QUOTE = "rdf:about=\"";
private static final String RDF_ABOUT_SINGLE_QUOTE = "rdf:about='";
private static final Logger log = LoggerFactory.getLogger(FilterInvalidRDF.class);
private final BufferedReader reader;
boolean initialized = false;
private byte[] currentLine = null;
......@@ -40,25 +46,49 @@ public class FilterInvalidRDF extends InputStream {
line = StringUtils.join(fragments, "&");
}
line = fixRdfAbout(line, "about");
return fixRdfAbout(line, "resource");
}
private static String fixRdfAbout(String line, String propertyName) {
final String propertyWithDoubleQuote = "rdf:" + propertyName + "=\"";
final String propertyWithSingleQuote = "rdf:" + propertyName + "='";
String before = null;
String iri = null;
String after = null;
if (line.contains(RDF_ABOUT_DOUBLE_QUOTE)) {
before = StringUtils.substringBefore(line, RDF_ABOUT_DOUBLE_QUOTE) + RDF_ABOUT_DOUBLE_QUOTE;
iri = StringUtils.substringBetween(line, RDF_ABOUT_DOUBLE_QUOTE, "\"");
after = "\"" + StringUtils.substringAfter(StringUtils.substringAfter(line, RDF_ABOUT_DOUBLE_QUOTE), "\"");
} else if (line.contains(RDF_ABOUT_SINGLE_QUOTE)) {
before = StringUtils.substringBefore(line, RDF_ABOUT_DOUBLE_QUOTE) + RDF_ABOUT_SINGLE_QUOTE;
iri = StringUtils.substringBetween(line, RDF_ABOUT_DOUBLE_QUOTE, "'");
after = "'" + StringUtils.substringAfter(StringUtils.substringAfter(line, RDF_ABOUT_DOUBLE_QUOTE), "'");
if (line.contains(propertyWithDoubleQuote)) {
before = StringUtils.substringBefore(line, propertyWithDoubleQuote) + propertyWithDoubleQuote;
iri = StringUtils.substringBetween(line, propertyWithDoubleQuote, "\"");
after = "\"" + StringUtils.substringAfter(StringUtils.substringAfter(line, propertyWithDoubleQuote), "\"");
} else if (line.contains(propertyWithSingleQuote)) {
before = StringUtils.substringBefore(line, propertyWithDoubleQuote) + propertyWithSingleQuote;
iri = StringUtils.substringBetween(line, propertyWithDoubleQuote, "'");
after = "'" + StringUtils.substringAfter(StringUtils.substringAfter(line, propertyWithDoubleQuote), "'");
}
if (iri == null) {
return line;
} else {
final String fixedIRI = iri.replaceAll(" ", "%20");
return before + fixedIRI + after;
return before + fixIRI(iri) + after;
}
}
private static String fixIRI(String iri) {
String result = iri.replaceAll(" ", "%20");
try {
IRIs.reference(result);
} catch (IRIException e) {
log.info("Found a really bad IRI: '{}'", iri);
// A reliable way to turn any text (even a broken URI) into a URI is to use a hash value.
result = "urn:" + DigestUtils.md5Hex(iri);
}
return result;
}
private void readNextLine() throws IOException {
......
package de.landsh.opendata.catalogproxy;
import org.apache.jena.rdf.model.*;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.ResIterator;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.riot.RDFLanguages;
import org.apache.jena.riot.RDFParser;
import org.apache.jena.riot.system.ErrorHandlerFactory;
......@@ -270,11 +273,53 @@ public class CatalogFilterTest {
try (final InputStream inputStream = getClass().getResourceAsStream("/catalog.xml")) {
final Model model = catalogFilter.work(inputStream);
Selector selectorSchema = new SimpleSelector(null, model.createProperty("http://schema.org/startDate"), (Object) null);
assertFalse(model.listStatements(selectorSchema).hasNext());
assertFalse(model.listStatements(null, model.createProperty("http://schema.org/startDate"), (String) null).hasNext());
assertTrue(model.listStatements(null, DCAT.startDate, (String) null).hasNext());
}
}
/**
* A data service must be preserved.
*/
@Test
public void dataservice() throws IOException {
try (final InputStream inputStream = getClass().getResourceAsStream("/dataservice.xml")) {
final Model model = catalogFilter.work(inputStream);
Selector selectorDCAT = new SimpleSelector(null, DCAT.startDate, (Object) null);
assertTrue(model.listStatements(selectorDCAT).hasNext());
assertEquals(1, model.listStatements(null, RDF.type, DCAT.Dataset).toList().size(), "one dcat:Dataset");
assertEquals(4, model.listStatements(null, RDF.type, DCAT.Distribution).toList().size(), "four dcat:Distribution");
assertEquals(2, model.listStatements(null, RDF.type, DCAT.DataService).toList().size(), "two dcat:DataService");
}
}
/**
* In case of a dcat:DataService no dcat:downloadURL must be added to a dcat:Distribution
* <a href="https://www.dcat-ap.de/def/dcatde/2.0/implRules/#konvention-41">Konvention 41</a> states that a
* dcat:Distribution MUST NOT have a dcat:downloadURL property.
*/
@Test
public void addDownloadURLs_DataService() {
final Model model = parseRdf(getClass().getResourceAsStream("/dataservice.xml"));
catalogFilter.addDownloadURLs(model);
Resource distributionWithService1 = model.getResource("https://umweltgeodienste.schleswig-holstein.de/WMS_UWAT_NAT?");
Resource distributionWithService2 = model.getResource("https://umweltgeodienste.schleswig-holstein.de/WFS_UWAT?");
Resource distributionWithoutService1 = model.getResource("https://umweltgeodienste.schleswig-holstein.de/WFS_UWAT?service=wfs&version=2.0.0&request=GetFeature&typeNames=app:ramsar");
Resource distributionWithoutService2 = model.getResource("https://opendata.schleswig-holstein.de/data/llur55/ramsar_utm32.zip");
assertTrue(distributionWithService1.hasProperty(DCAT.accessURL));
assertTrue(distributionWithService1.hasProperty(DCAT.accessService));
assertFalse(distributionWithService1.hasProperty(DCAT.downloadURL), "a dcat:Distribution with a dcat:DataService MUST NOT have a dcat:downloadURL");
assertTrue(distributionWithService2.hasProperty(DCAT.accessURL));
assertTrue(distributionWithService2.hasProperty(DCAT.accessService));
assertFalse(distributionWithService2.hasProperty(DCAT.downloadURL), "a dcat:Distribution with a dcat:DataService MUST NOT have a dcat:downloadURL");
assertTrue(distributionWithoutService1.hasProperty(DCAT.accessURL));
assertTrue(distributionWithoutService1.hasProperty(DCAT.downloadURL));
assertTrue(distributionWithoutService2.hasProperty(DCAT.accessURL));
assertTrue(distributionWithoutService2.hasProperty(DCAT.downloadURL));
}
}
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment