Merge branch '8-dataservice-und-datasetseries-erhalten' into 'master'

Resolve "DataService und DatasetSeries erhalten"

Closes #8

See merge request !5

pom.xml

 <?xml version="1.0" encoding="utf-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.springframework.boot</groupId>
     <artifactId>spring-boot-starter-parent</artifactId>
-    <version>3.2.2</version>
+    <version>3.4.0</version>
     <relativePath />
   </parent>
   <groupId>de.landsh.opendata</groupId>
   <artifactId>dcat-catalog-proxy</artifactId>
-  <version>1.3.0</version>
+  <version>1.4.0-SNAPSHOT</version>
   <name>dcat-catalog-proxy</name>
   <description>DCAT catalog proxy</description>
   <properties>
     <java.version>17</java.version>
-    <jena.version>4.7.0</jena.version>
+    <jena.version>5.2.0</jena.version>
   </properties>
   <dependencies>
     <dependency>
@@ -80,6 +79,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
         <artifactId>snakeyaml</artifactId>
         <version>1.32</version>
       </dependency>
+      <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
+      <dependency>
+        <groupId>commons-io</groupId>
+        <artifactId>commons-io</artifactId>
+        <version>2.18.0</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   <build>
@@ -92,7 +97,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
       <plugin>
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
-        <version>0.8.7</version>
+        <version>0.8.12</version>
         <executions>
           <execution>
             <id>coverage-initialize</id>
@@ -157,7 +162,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
       <plugin>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-maven</artifactId>
-        <version>7.4.4</version>
+        <version>11.1.0</version>
         <configuration>
           <suppressionFiles>
             <suppressionFile>owasp-suppressions.xml</suppressionFile>
@@ -165,6 +170,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
           <failBuildOnCVSS>8</failBuildOnCVSS>
           <assemblyAnalyzerEnabled>false</assemblyAnalyzerEnabled>
           <failOnError>true</failOnError>
+          <nvdApiKey>${env.NVD_API_KEY}</nvdApiKey>
         </configuration>
         <executions>
           <execution>
@@ -177,7 +183,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>versions-maven-plugin</artifactId>
-        <version>2.14.2</version>
+        <version>2.18.0</version>
         <configuration>
           <rulesUri>file:///${project.basedir}/rules.xml</rulesUri>
         </configuration>
@@ -187,14 +193,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
         <artifactId>spotbugs-maven-plugin</artifactId>
         <version>4.5.3.0</version>
       </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>versions-maven-plugin</artifactId>
-        <version>2.14.2</version>
-        <configuration>
-          <rulesUri>file:///${project.basedir}/rules.xml</rulesUri>
-        </configuration>
-      </plugin>
     </plugins>
   </build>
 </project>

rules.xml

@@ -11,15 +11,5 @@
     <ignoreVersion type="regex">(?i).*M(?:-?\d+)?</ignoreVersion>
   </ignoreVersions>
   <rules>
-
-    <!-- 4.7.0 is the last version that work. There is an additional rule for
-         IRIs in later versions that prevents some of CKAN's responses
-         to be parsed.  -->
-    <rule groupId="org.apache.jena" comparisonMethod="maven">
-      <ignoreVersions>
-        <ignoreVersion type="regex">4.[89].*</ignoreVersion>
-        <ignoreVersion type="regex">4.1[0-9].*</ignoreVersion>
-      </ignoreVersions>
-    </rule>
   </rules>
 </ruleset>

src/main/java/de/landsh/opendata/catalogproxy/CatalogFilter.java

@@ -14,6 +14,7 @@ import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 
 import java.io.InputStream;
+import java.nio.channels.Selector;
 import java.util.*;
 
 /**
@@ -129,8 +130,9 @@ public class CatalogFilter implements InitializingBean {
 
             final Resource accessURL = distribution.getPropertyResourceValue(DCAT.accessURL);
             final Resource downloadURL = distribution.getPropertyResourceValue(DCAT.downloadURL);
+            final boolean hasAccessService = distribution.hasProperty(DCAT.accessService);
 
-            if (downloadURL == null) {
+            if (downloadURL == null && !hasAccessService) {
                 distribution.addProperty(DCAT.downloadURL, accessURL);
             }
         }
@@ -359,8 +361,7 @@ public class CatalogFilter implements InitializingBean {
     void changeStartAndEndDate(Model model) {
         LinkedList<Statement> oldStatements = new LinkedList<>();
 
-        Selector selectorStart = new SimpleSelector(null, model.createProperty("http://schema.org/startDate"), (Object) null);
-        StmtIterator it = model.listStatements(selectorStart);
+        StmtIterator it = model.listStatements(null, model.createProperty("http://schema.org/startDate"), (String) null);
         while (it.hasNext()) {
             Statement stmt = it.next();
             oldStatements.add(stmt);
@@ -368,8 +369,7 @@ public class CatalogFilter implements InitializingBean {
         }
         it.close();
 
-        Selector selectorEnd = new SimpleSelector(null, model.createProperty("http://schema.org/endDate"), (Object) null);
-        it = model.listStatements(selectorEnd);
+        it = model.listStatements(null, model.createProperty("http://schema.org/endDate"), (String) null);
         while (it.hasNext()) {
             Statement stmt = it.next();
             oldStatements.add(stmt);

src/main/java/de/landsh/opendata/catalogproxy/FilterInvalidRDF.java

 package de.landsh.opendata.catalogproxy;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.jena.irix.IRIException;
+import org.apache.jena.irix.IRIs;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -10,8 +15,9 @@ import java.nio.charset.StandardCharsets;
 
 public class FilterInvalidRDF extends InputStream {
 
-    private static final String RDF_ABOUT_DOUBLE_QUOTE = "rdf:about=\"";
-    private static final String RDF_ABOUT_SINGLE_QUOTE = "rdf:about='";
+
+    private static final Logger log = LoggerFactory.getLogger(FilterInvalidRDF.class);
+
     private final BufferedReader reader;
     boolean initialized = false;
     private byte[] currentLine = null;
@@ -40,25 +46,49 @@ public class FilterInvalidRDF extends InputStream {
             line = StringUtils.join(fragments, "&");
         }
 
+        line = fixRdfAbout(line, "about");
+        return fixRdfAbout(line, "resource");
+    }
+
+    private static String fixRdfAbout(String line, String propertyName) {
+        final String propertyWithDoubleQuote = "rdf:" + propertyName + "=\"";
+        final String propertyWithSingleQuote = "rdf:" + propertyName + "='";
+
         String before = null;
         String iri = null;
         String after = null;
-        if (line.contains(RDF_ABOUT_DOUBLE_QUOTE)) {
-            before = StringUtils.substringBefore(line, RDF_ABOUT_DOUBLE_QUOTE) + RDF_ABOUT_DOUBLE_QUOTE;
-            iri = StringUtils.substringBetween(line, RDF_ABOUT_DOUBLE_QUOTE, "\"");
-            after = "\"" + StringUtils.substringAfter(StringUtils.substringAfter(line, RDF_ABOUT_DOUBLE_QUOTE), "\"");
-        } else if (line.contains(RDF_ABOUT_SINGLE_QUOTE)) {
-            before = StringUtils.substringBefore(line, RDF_ABOUT_DOUBLE_QUOTE) + RDF_ABOUT_SINGLE_QUOTE;
-            iri = StringUtils.substringBetween(line, RDF_ABOUT_DOUBLE_QUOTE, "'");
-            after = "'" + StringUtils.substringAfter(StringUtils.substringAfter(line, RDF_ABOUT_DOUBLE_QUOTE), "'");
+        if (line.contains(propertyWithDoubleQuote)) {
+            before = StringUtils.substringBefore(line, propertyWithDoubleQuote) + propertyWithDoubleQuote;
+            iri = StringUtils.substringBetween(line, propertyWithDoubleQuote, "\"");
+            after = "\"" + StringUtils.substringAfter(StringUtils.substringAfter(line, propertyWithDoubleQuote), "\"");
+        } else if (line.contains(propertyWithSingleQuote)) {
+            before = StringUtils.substringBefore(line, propertyWithDoubleQuote) + propertyWithSingleQuote;
+            iri = StringUtils.substringBetween(line, propertyWithDoubleQuote, "'");
+            after = "'" + StringUtils.substringAfter(StringUtils.substringAfter(line, propertyWithDoubleQuote), "'");
         }
 
         if (iri == null) {
             return line;
         } else {
-            final String fixedIRI = iri.replaceAll(" ", "%20");
-            return before + fixedIRI + after;
+
+            return before + fixIRI(iri) + after;
+        }
     }
+
+    private static String fixIRI(String iri) {
+        String result = iri.replaceAll(" ", "%20");
+
+        try {
+            IRIs.reference(result);
+        } catch (IRIException e) {
+            log.info("Found a really bad IRI: '{}'", iri);
+
+            // A reliable way to turn any text (even a broken URI) into a URI is to use a hash value.
+
+            result = "urn:" + DigestUtils.md5Hex(iri);
+        }
+
+        return result;
     }
 
     private void readNextLine() throws IOException {

src/test/java/de/landsh/opendata/catalogproxy/CatalogFilterTest.java

 package de.landsh.opendata.catalogproxy;
 
-import org.apache.jena.rdf.model.*;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.ResIterator;
+import org.apache.jena.rdf.model.Resource;
 import org.apache.jena.riot.RDFLanguages;
 import org.apache.jena.riot.RDFParser;
 import org.apache.jena.riot.system.ErrorHandlerFactory;
@@ -270,11 +273,53 @@ public class CatalogFilterTest {
         try (final InputStream inputStream = getClass().getResourceAsStream("/catalog.xml")) {
             final Model model = catalogFilter.work(inputStream);
 
-            Selector selectorSchema = new SimpleSelector(null, model.createProperty("http://schema.org/startDate"), (Object) null);
-            assertFalse(model.listStatements(selectorSchema).hasNext());
+            assertFalse(model.listStatements(null, model.createProperty("http://schema.org/startDate"), (String) null).hasNext());
+            assertTrue(model.listStatements(null, DCAT.startDate, (String) null).hasNext());
+        }
+    }
+
+    /**
+     * A data service must be preserved.
+     */
+    @Test
+    public void dataservice() throws IOException {
+        try (final InputStream inputStream = getClass().getResourceAsStream("/dataservice.xml")) {
+            final Model model = catalogFilter.work(inputStream);
 
-            Selector selectorDCAT = new SimpleSelector(null, DCAT.startDate, (Object) null);
-            assertTrue(model.listStatements(selectorDCAT).hasNext());
+            assertEquals(1, model.listStatements(null, RDF.type, DCAT.Dataset).toList().size(), "one dcat:Dataset");
+            assertEquals(4, model.listStatements(null, RDF.type, DCAT.Distribution).toList().size(), "four dcat:Distribution");
+            assertEquals(2, model.listStatements(null, RDF.type, DCAT.DataService).toList().size(), "two dcat:DataService");
+        }
     }
+
+    /**
+     * In case of a dcat:DataService no dcat:downloadURL must be added to a dcat:Distribution
+     * <a href="https://www.dcat-ap.de/def/dcatde/2.0/implRules/#konvention-41">Konvention 41</a> states that a
+     * dcat:Distribution MUST NOT have a dcat:downloadURL property.
+     */
+    @Test
+    public void addDownloadURLs_DataService() {
+        final Model model = parseRdf(getClass().getResourceAsStream("/dataservice.xml"));
+
+        catalogFilter.addDownloadURLs(model);
+
+        Resource distributionWithService1 = model.getResource("https://umweltgeodienste.schleswig-holstein.de/WMS_UWAT_NAT?");
+        Resource distributionWithService2 = model.getResource("https://umweltgeodienste.schleswig-holstein.de/WFS_UWAT?");
+        Resource distributionWithoutService1 = model.getResource("https://umweltgeodienste.schleswig-holstein.de/WFS_UWAT?service=wfs&version=2.0.0&request=GetFeature&typeNames=app:ramsar");
+        Resource distributionWithoutService2 = model.getResource("https://opendata.schleswig-holstein.de/data/llur55/ramsar_utm32.zip");
+
+        assertTrue(distributionWithService1.hasProperty(DCAT.accessURL));
+        assertTrue(distributionWithService1.hasProperty(DCAT.accessService));
+        assertFalse(distributionWithService1.hasProperty(DCAT.downloadURL), "a dcat:Distribution with a dcat:DataService MUST NOT have a dcat:downloadURL");
+
+        assertTrue(distributionWithService2.hasProperty(DCAT.accessURL));
+        assertTrue(distributionWithService2.hasProperty(DCAT.accessService));
+        assertFalse(distributionWithService2.hasProperty(DCAT.downloadURL), "a dcat:Distribution with a dcat:DataService MUST NOT have a dcat:downloadURL");
+
+        assertTrue(distributionWithoutService1.hasProperty(DCAT.accessURL));
+        assertTrue(distributionWithoutService1.hasProperty(DCAT.downloadURL));
+        
+        assertTrue(distributionWithoutService2.hasProperty(DCAT.accessURL));
+        assertTrue(distributionWithoutService2.hasProperty(DCAT.downloadURL));
     }
 }