Skip to content
Snippets Groups Projects
Verified Commit dca021a7 authored by Jesper Zedlitz's avatar Jesper Zedlitz
Browse files

test that processes a real DCAT dataset served by a local HTTP server

parent 88294ce5
Branches
Tags
1 merge request!1Update Formats, Dependencies, and Dockerfile Configuration
@prefix dcat: <http://www.w3.org/ns/dcat#> .
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
<https://example.org/dataset/c0b08e27-c2a1-4602-83fc-961f0f50190c> a dcat:Dataset ;
dcterms:identifier "c0b08e27-c2a1-4602-83fc-961f0f50190c" ;
dcterms:title "all test files" ;
dcterms:publisher [
a foaf:Organization;
foaf:name "Test" ;
] ;
dcat:distribution [ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/ATOM> ;
dcat:downloadURL <http://localhost:8000/tests/data/Atom_SH_Feldblockfinder_OpenGBD.xml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/TXT> ;
dcat:downloadURL <http://localhost:8000/tests/data/text.txt> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/CSV> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo.csv> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GEOJSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda.geojson> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo-resource.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo-schema.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/RDF> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo.ttl> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/DOCX> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.docx> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GEOTIFF> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid_geotiff.tif> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/ODS> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.ods> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/ODT> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.odt> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/XML> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.xml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/PARQUET> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.parquet> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/TIFF> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.tif> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/XLSX> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.xlsx> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GML> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda.gml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/WMTS_SRVC> ;
dcat:downloadURL <http://localhost:8000/tests/data/WMTSCapabilities.xml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/SHP> ;
dcat:downloadURL <http://localhost:8000/tests/data/zos116.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/SHP> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda-with-subdir.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/SHP> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json.bz2> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GEOTIFF> ;
dcat:downloadURL <http://localhost:8000/tests/data/geotiff.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:compressFormat <http://www.iana.org/assignments/media-types/application/gzip> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json.gz> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json.xz> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/PNG> ;
dcat:downloadURL <http://localhost:8000/tests/data/image.png> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/json-in-zip.zip> ;
dcat:packageFormat <http://publications.europa.eu/resource/authority/file-type/ZIP> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/jsons-in-zip.zip> ;
dcat:packageFormat <http://publications.europa.eu/resource/authority/file-type/ZIP> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/PNG> ;
dcat:downloadURL <http://localhost:8000/tests/data/png-in-zip.zip> ;
dcat:packageFormat <http://publications.europa.eu/resource/authority/file-type/ZIP> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/RDF> ;
dcat:downloadURL <http://localhost:8000/tests/data/rdf.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/RDF> ;
dcat:downloadURL <http://localhost:8000/tests/data/rdf.xml> ] .
...@@ -7,6 +7,12 @@ from dcat_catalog_check import ( ...@@ -7,6 +7,12 @@ from dcat_catalog_check import (
) )
from rdflib import Graph from rdflib import Graph
from rdflib.namespace import RDF, DCAT from rdflib.namespace import RDF, DCAT
from http.server import SimpleHTTPRequestHandler, HTTPServer
import threading
import time
import io
import requests
import sys
class TestDcatCatalogCheck(unittest.TestCase): class TestDcatCatalogCheck(unittest.TestCase):
...@@ -29,13 +35,10 @@ class TestDcatCatalogCheck(unittest.TestCase): ...@@ -29,13 +35,10 @@ class TestDcatCatalogCheck(unittest.TestCase):
"XML": ["application/xml"], "XML": ["application/xml"],
} }
self.assertTrue(self.dcc.is_mime_type_compatible( self.assertTrue(self.dcc.is_mime_type_compatible("JSON", "application/json"))
"JSON", "application/json")) self.assertFalse(self.dcc.is_mime_type_compatible("JSON", "application/xml"))
self.assertFalse(self.dcc.is_mime_type_compatible(
"JSON", "application/xml"))
self.assertFalse( self.assertFalse(
self.dcc.is_mime_type_compatible( self.dcc.is_mime_type_compatible("UnknownFormat", "application/json")
"UnknownFormat", "application/json")
) )
def test_read_allowed_file_formats(self): def test_read_allowed_file_formats(self):
...@@ -47,8 +50,7 @@ class TestDcatCatalogCheck(unittest.TestCase): ...@@ -47,8 +50,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
): ):
formats = self.dcc.read_allowed_file_formats() formats = self.dcc.read_allowed_file_formats()
self.assertEqual( self.assertEqual(
formats, {"JSON": ["application/json"], formats, {"JSON": ["application/json"], "XML": ["application/xml"]}
"XML": ["application/xml"]}
) )
def test_load_uri_replacements(self): def test_load_uri_replacements(self):
...@@ -59,8 +61,7 @@ class TestDcatCatalogCheck(unittest.TestCase): ...@@ -59,8 +61,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
), ),
): ):
replacements = self.dcc.load_uri_replacements() replacements = self.dcc.load_uri_replacements()
self.assertEqual( self.assertEqual(replacements, [{"regex": "old", "replaced_by": "new"}])
replacements, [{"regex": "old", "replaced_by": "new"}])
# Simulate that the file does not exist # Simulate that the file does not exist
...@@ -456,6 +457,41 @@ class TestDcatCatalogCheck(unittest.TestCase): ...@@ -456,6 +457,41 @@ class TestDcatCatalogCheck(unittest.TestCase):
self.assertEqual(resource["url"], "http://example.com/data") self.assertEqual(resource["url"], "http://example.com/data")
self.assertEqual(resource["format"], "JSON") self.assertEqual(resource["format"], "JSON")
def _wait_for_server(self, url, timeout=10, interval=0.2):
"""Wait until the server can be reached at the specified URL."""
start_time = time.time()
while time.time() - start_time < timeout:
try:
response = requests.get(url)
if response.status_code == 200:
return True
except requests.exceptions.RequestException:
pass
time.sleep(interval)
print(f"Timeout reached: Server at {url} not reachable.")
return False
def _run_server(self):
server_address = ("", 8000)
httpd = HTTPServer(server_address, SimpleHTTPRequestHandler)
httpd.serve_forever()
def test_read_dcat_catalog(self):
server_thread = threading.Thread(target=self._run_server)
server_thread.daemon = True
server_thread.start()
self._wait_for_server("http://localhost:8000")
mock_stdout = io.StringIO()
sys.stdout = mock_stdout
self.dcc.read_dcat_catalog("http://localhost:8000/tests/data/all-tests.ttl")
sys.stdout = sys.__stdout__
output = mock_stdout.getvalue()
json_objects = [json.loads(line) for line in output.splitlines()]
self.assertEqual(len(json_objects), 31)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment