Skip to content
Snippets Groups Projects
Verified Commit dca021a7 authored by Jesper Zedlitz's avatar Jesper Zedlitz
Browse files

test that processes a real DCAT dataset served by a local HTTP server

parent 88294ce5
Branches
Tags
1 merge request!1Update Formats, Dependencies, and Dockerfile Configuration
@prefix dcat: <http://www.w3.org/ns/dcat#> .
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
<https://example.org/dataset/c0b08e27-c2a1-4602-83fc-961f0f50190c> a dcat:Dataset ;
dcterms:identifier "c0b08e27-c2a1-4602-83fc-961f0f50190c" ;
dcterms:title "all test files" ;
dcterms:publisher [
a foaf:Organization;
foaf:name "Test" ;
] ;
dcat:distribution [ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/ATOM> ;
dcat:downloadURL <http://localhost:8000/tests/data/Atom_SH_Feldblockfinder_OpenGBD.xml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/TXT> ;
dcat:downloadURL <http://localhost:8000/tests/data/text.txt> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/CSV> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo.csv> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GEOJSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda.geojson> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo-resource.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo-schema.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/RDF> ;
dcat:downloadURL <http://localhost:8000/tests/data/ufo.ttl> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/DOCX> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.docx> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GEOTIFF> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid_geotiff.tif> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/ODS> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.ods> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/ODT> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.odt> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/XML> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.xml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/PARQUET> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.parquet> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/TIFF> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.tif> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/XLSX> ;
dcat:downloadURL <http://localhost:8000/tests/data/valid.xlsx> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GML> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda.gml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/WMTS_SRVC> ;
dcat:downloadURL <http://localhost:8000/tests/data/WMTSCapabilities.xml> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/SHP> ;
dcat:downloadURL <http://localhost:8000/tests/data/zos116.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/SHP> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda-with-subdir.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/SHP> ;
dcat:downloadURL <http://localhost:8000/tests/data/bermuda.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json.bz2> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/GEOTIFF> ;
dcat:downloadURL <http://localhost:8000/tests/data/geotiff.zip> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:compressFormat <http://www.iana.org/assignments/media-types/application/gzip> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json.gz> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/correct.json.xz> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/PNG> ;
dcat:downloadURL <http://localhost:8000/tests/data/image.png> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/json-in-zip.zip> ;
dcat:packageFormat <http://publications.europa.eu/resource/authority/file-type/ZIP> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/JSON> ;
dcat:downloadURL <http://localhost:8000/tests/data/jsons-in-zip.zip> ;
dcat:packageFormat <http://publications.europa.eu/resource/authority/file-type/ZIP> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/PNG> ;
dcat:downloadURL <http://localhost:8000/tests/data/png-in-zip.zip> ;
dcat:packageFormat <http://publications.europa.eu/resource/authority/file-type/ZIP> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/RDF> ;
dcat:downloadURL <http://localhost:8000/tests/data/rdf.json> ],
[ a dcat:Distribution ;
dcterms:format <http://publications.europa.eu/resource/authority/file-type/RDF> ;
dcat:downloadURL <http://localhost:8000/tests/data/rdf.xml> ] .
......@@ -7,6 +7,12 @@ from dcat_catalog_check import (
)
from rdflib import Graph
from rdflib.namespace import RDF, DCAT
from http.server import SimpleHTTPRequestHandler, HTTPServer
import threading
import time
import io
import requests
import sys
class TestDcatCatalogCheck(unittest.TestCase):
......@@ -29,13 +35,10 @@ class TestDcatCatalogCheck(unittest.TestCase):
"XML": ["application/xml"],
}
self.assertTrue(self.dcc.is_mime_type_compatible(
"JSON", "application/json"))
self.assertFalse(self.dcc.is_mime_type_compatible(
"JSON", "application/xml"))
self.assertTrue(self.dcc.is_mime_type_compatible("JSON", "application/json"))
self.assertFalse(self.dcc.is_mime_type_compatible("JSON", "application/xml"))
self.assertFalse(
self.dcc.is_mime_type_compatible(
"UnknownFormat", "application/json")
self.dcc.is_mime_type_compatible("UnknownFormat", "application/json")
)
def test_read_allowed_file_formats(self):
......@@ -47,8 +50,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
):
formats = self.dcc.read_allowed_file_formats()
self.assertEqual(
formats, {"JSON": ["application/json"],
"XML": ["application/xml"]}
formats, {"JSON": ["application/json"], "XML": ["application/xml"]}
)
def test_load_uri_replacements(self):
......@@ -59,8 +61,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
),
):
replacements = self.dcc.load_uri_replacements()
self.assertEqual(
replacements, [{"regex": "old", "replaced_by": "new"}])
self.assertEqual(replacements, [{"regex": "old", "replaced_by": "new"}])
# Simulate that the file does not exist
......@@ -456,6 +457,41 @@ class TestDcatCatalogCheck(unittest.TestCase):
self.assertEqual(resource["url"], "http://example.com/data")
self.assertEqual(resource["format"], "JSON")
def _wait_for_server(self, url, timeout=10, interval=0.2):
"""Wait until the server can be reached at the specified URL."""
start_time = time.time()
while time.time() - start_time < timeout:
try:
response = requests.get(url)
if response.status_code == 200:
return True
except requests.exceptions.RequestException:
pass
time.sleep(interval)
print(f"Timeout reached: Server at {url} not reachable.")
return False
def _run_server(self):
server_address = ("", 8000)
httpd = HTTPServer(server_address, SimpleHTTPRequestHandler)
httpd.serve_forever()
def test_read_dcat_catalog(self):
server_thread = threading.Thread(target=self._run_server)
server_thread.daemon = True
server_thread.start()
self._wait_for_server("http://localhost:8000")
mock_stdout = io.StringIO()
sys.stdout = mock_stdout
self.dcc.read_dcat_catalog("http://localhost:8000/tests/data/all-tests.ttl")
sys.stdout = sys.__stdout__
output = mock_stdout.getvalue()
json_objects = [json.loads(line) for line in output.splitlines()]
self.assertEqual(len(json_objects), 31)
if __name__ == "__main__":
unittest.main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment