Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
DCAT Catalog Check
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Open-Data
DCAT Catalog Check
Commits
dca021a7
Verified
Commit
dca021a7
authored
4 months ago
by
Jesper Zedlitz
Browse files
Options
Downloads
Patches
Plain Diff
test that processes a real DCAT dataset served by a local HTTP server
parent
88294ce5
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!1
Update Formats, Dependencies, and Dockerfile Configuration
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
tests/data/all-tests.ttl
+109
-0
109 additions, 0 deletions
tests/data/all-tests.ttl
tests/test_dcat_catalog_check.py
+46
-10
46 additions, 10 deletions
tests/test_dcat_catalog_check.py
with
155 additions
and
10 deletions
tests/data/all-tests.ttl
0 → 100644
+
109
−
0
View file @
dca021a7
@prefix
dcat:
<http://www.w3.org/ns/dcat#>
.
@prefix
dcterms:
<http://purl.org/dc/terms/>
.
@prefix
foaf:
<http://xmlns.com/foaf/0.1/>
.
<https://example.org/dataset/c0b08e27-c2a1-4602-83fc-961f0f50190c>
a
dcat:
Dataset
;
dcterms:
identifier
"c0b08e27-c2a1-4602-83fc-961f0f50190c"
;
dcterms:
title
"all test files"
;
dcterms:
publisher
[
a
foaf:
Organization
;
foaf:
name
"Test"
;
]
;
dcat:
distribution
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/ATOM>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/Atom_SH_Feldblockfinder_OpenGBD.xml>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/TXT>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/text.txt>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/CSV>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/ufo.csv>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/GEOJSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/bermuda.geojson>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/ufo-resource.json>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/ufo-schema.json>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/RDF>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/ufo.ttl>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/DOCX>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid.docx>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/GEOTIFF>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid_geotiff.tif>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/ODS>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid.ods>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/ODT>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid.odt>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/XML>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/correct.xml>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/PARQUET>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid.parquet>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/TIFF>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid.tif>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/XLSX>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/valid.xlsx>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/GML>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/bermuda.gml>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/WMTS_SRVC>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/WMTSCapabilities.xml>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/SHP>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/zos116.zip>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/SHP>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/bermuda-with-subdir.zip>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/SHP>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/bermuda.zip>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/correct.json>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/correct.json.bz2>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/GEOTIFF>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/geotiff.zip>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
compressFormat
<http://www.iana.org/assignments/media-types/application/gzip>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/correct.json.gz>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/correct.json.xz>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/PNG>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/image.png>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/json-in-zip.zip>
;
dcat:
packageFormat
<http://publications.europa.eu/resource/authority/file-type/ZIP>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/JSON>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/jsons-in-zip.zip>
;
dcat:
packageFormat
<http://publications.europa.eu/resource/authority/file-type/ZIP>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/PNG>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/png-in-zip.zip>
;
dcat:
packageFormat
<http://publications.europa.eu/resource/authority/file-type/ZIP>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/RDF>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/rdf.json>
],
[
a
dcat:
Distribution
;
dcterms:
format
<http://publications.europa.eu/resource/authority/file-type/RDF>
;
dcat:
downloadURL
<http://localhost:8000/tests/data/rdf.xml>
]
.
This diff is collapsed.
Click to expand it.
tests/test_
format_fidelity
_check
er
.py
→
tests/test_
dcat_catalog
_check.py
+
46
−
10
View file @
dca021a7
...
@@ -7,6 +7,12 @@ from dcat_catalog_check import (
...
@@ -7,6 +7,12 @@ from dcat_catalog_check import (
)
)
from
rdflib
import
Graph
from
rdflib
import
Graph
from
rdflib.namespace
import
RDF
,
DCAT
from
rdflib.namespace
import
RDF
,
DCAT
from
http.server
import
SimpleHTTPRequestHandler
,
HTTPServer
import
threading
import
time
import
io
import
requests
import
sys
class
TestDcatCatalogCheck
(
unittest
.
TestCase
):
class
TestDcatCatalogCheck
(
unittest
.
TestCase
):
...
@@ -29,13 +35,10 @@ class TestDcatCatalogCheck(unittest.TestCase):
...
@@ -29,13 +35,10 @@ class TestDcatCatalogCheck(unittest.TestCase):
"
XML
"
:
[
"
application/xml
"
],
"
XML
"
:
[
"
application/xml
"
],
}
}
self
.
assertTrue
(
self
.
dcc
.
is_mime_type_compatible
(
self
.
assertTrue
(
self
.
dcc
.
is_mime_type_compatible
(
"
JSON
"
,
"
application/json
"
))
"
JSON
"
,
"
application/json
"
))
self
.
assertFalse
(
self
.
dcc
.
is_mime_type_compatible
(
"
JSON
"
,
"
application/xml
"
))
self
.
assertFalse
(
self
.
dcc
.
is_mime_type_compatible
(
"
JSON
"
,
"
application/xml
"
))
self
.
assertFalse
(
self
.
assertFalse
(
self
.
dcc
.
is_mime_type_compatible
(
self
.
dcc
.
is_mime_type_compatible
(
"
UnknownFormat
"
,
"
application/json
"
)
"
UnknownFormat
"
,
"
application/json
"
)
)
)
def
test_read_allowed_file_formats
(
self
):
def
test_read_allowed_file_formats
(
self
):
...
@@ -47,8 +50,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
...
@@ -47,8 +50,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
):
):
formats
=
self
.
dcc
.
read_allowed_file_formats
()
formats
=
self
.
dcc
.
read_allowed_file_formats
()
self
.
assertEqual
(
self
.
assertEqual
(
formats
,
{
"
JSON
"
:
[
"
application/json
"
],
formats
,
{
"
JSON
"
:
[
"
application/json
"
],
"
XML
"
:
[
"
application/xml
"
]}
"
XML
"
:
[
"
application/xml
"
]}
)
)
def
test_load_uri_replacements
(
self
):
def
test_load_uri_replacements
(
self
):
...
@@ -59,8 +61,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
...
@@ -59,8 +61,7 @@ class TestDcatCatalogCheck(unittest.TestCase):
),
),
):
):
replacements
=
self
.
dcc
.
load_uri_replacements
()
replacements
=
self
.
dcc
.
load_uri_replacements
()
self
.
assertEqual
(
self
.
assertEqual
(
replacements
,
[{
"
regex
"
:
"
old
"
,
"
replaced_by
"
:
"
new
"
}])
replacements
,
[{
"
regex
"
:
"
old
"
,
"
replaced_by
"
:
"
new
"
}])
# Simulate that the file does not exist
# Simulate that the file does not exist
...
@@ -456,6 +457,41 @@ class TestDcatCatalogCheck(unittest.TestCase):
...
@@ -456,6 +457,41 @@ class TestDcatCatalogCheck(unittest.TestCase):
self
.
assertEqual
(
resource
[
"
url
"
],
"
http://example.com/data
"
)
self
.
assertEqual
(
resource
[
"
url
"
],
"
http://example.com/data
"
)
self
.
assertEqual
(
resource
[
"
format
"
],
"
JSON
"
)
self
.
assertEqual
(
resource
[
"
format
"
],
"
JSON
"
)
def
_wait_for_server
(
self
,
url
,
timeout
=
10
,
interval
=
0.2
):
"""
Wait until the server can be reached at the specified URL.
"""
start_time
=
time
.
time
()
while
time
.
time
()
-
start_time
<
timeout
:
try
:
response
=
requests
.
get
(
url
)
if
response
.
status_code
==
200
:
return
True
except
requests
.
exceptions
.
RequestException
:
pass
time
.
sleep
(
interval
)
print
(
f
"
Timeout reached: Server at
{
url
}
not reachable.
"
)
return
False
def
_run_server
(
self
):
server_address
=
(
""
,
8000
)
httpd
=
HTTPServer
(
server_address
,
SimpleHTTPRequestHandler
)
httpd
.
serve_forever
()
def
test_read_dcat_catalog
(
self
):
server_thread
=
threading
.
Thread
(
target
=
self
.
_run_server
)
server_thread
.
daemon
=
True
server_thread
.
start
()
self
.
_wait_for_server
(
"
http://localhost:8000
"
)
mock_stdout
=
io
.
StringIO
()
sys
.
stdout
=
mock_stdout
self
.
dcc
.
read_dcat_catalog
(
"
http://localhost:8000/tests/data/all-tests.ttl
"
)
sys
.
stdout
=
sys
.
__stdout__
output
=
mock_stdout
.
getvalue
()
json_objects
=
[
json
.
loads
(
line
)
for
line
in
output
.
splitlines
()]
self
.
assertEqual
(
len
(
json_objects
),
31
)
if
__name__
==
"
__main__
"
:
if
__name__
==
"
__main__
"
:
unittest
.
main
()
unittest
.
main
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment