From 7a80b6699ab7cff7154a7bd4ed3316f06a6cfbd0 Mon Sep 17 00:00:00 2001
From: Thorge Petersen <petersen@rz.uni-kiel.de>
Date: Thu, 8 Jun 2023 12:51:53 +0200
Subject: [PATCH] Reorganized default mapping and resource files
---
CHANGELOG.md | 12 ++++++-
README.md | 33 +++++++++++--------
ckanext/odsh/helpers.py | 18 +++++-----
ckanext/odsh/helpers_tpsh.py | 14 +++++---
ckanext/odsh/logic/action.py | 2 +-
ckanext/odsh/precondition.py | 5 ---
.../profiles/odsh_european_dcatap_profile.py | 11 ++++---
.../odsh/resources/fileformats.rdf | 0
.../odsh/resources/language_mapping.json | 0
.../odsh/resources/languages.json | 0
.../odsh/resources/licenses.json | 0
.../odsh/resources/mdk_mapping.json | 0
.../resource_format_openness_scores.json | 0
.../resources/schleswig-holstein_geojson.csv | 0
.../odsh/resources/subject_mapping.json | 0
ckanext/odsh/tests/test_env.py | 4 +--
ckanext/odsh/tests/test_helpers.py | 2 +-
ckanext/odsh/tests_tpsh/test_helpers_tpsh.py | 2 +-
ckanext/odsh/validation.py | 8 +++--
19 files changed, 67 insertions(+), 44 deletions(-)
delete mode 100644 ckanext/odsh/precondition.py
rename fileformats.rdf => ckanext/odsh/resources/fileformats.rdf (100%)
rename language_mapping.json => ckanext/odsh/resources/language_mapping.json (100%)
rename languages.json => ckanext/odsh/resources/languages.json (100%)
rename licenses.json => ckanext/odsh/resources/licenses.json (100%)
rename mdk_mapping.json => ckanext/odsh/resources/mdk_mapping.json (100%)
rename ckanext/odsh/{ => resources}/resource_format_openness_scores.json (100%)
rename schleswig-holstein_geojson.csv => ckanext/odsh/resources/schleswig-holstein_geojson.csv (100%)
rename subject_mapping.json => ckanext/odsh/resources/subject_mapping.json (100%)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f136f11a..456d020a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,17 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [2.0.0]
+## [Unreleased]
+
+### Removed
+
+- Removed Travis CI 'bin/' directory, along with all associated Travis CI configuration files and scripts.
+
+### Changed
+
+- Reorganized default mapping and resource files for improved organization and maintenance. Mapping and config files (e.g., file formats, licenses subject mappings, etc.) are now located in `ckanext/odsh/resources/`.
+
+## [2.0.0] - 2023-06-08
### Added
diff --git a/README.md b/README.md
index 4bc711a8..7e33b1bb 100644
--- a/README.md
+++ b/README.md
@@ -87,20 +87,25 @@ the config file is located at `/etc/ckan/default/production.ini`).
The extension requires configuration parameters in the CKAN configuration file (e.g.,
`production.ini`). The following parameters should be set:
-Parameter | Type | Description
---------- | ----- | -----------
-ckanext.odsh.home | `string` | Absolute path to base directory, e.g., `/usr/lib/ckan/default/src/ckanext-odsh`.
-ckanext.odsh.public_url | `string` | The public URL, e.g., `https://opendata.schleswig-holstein.de`.
-ckanext.odsh.showtestbanner | `boolean` | `true` switches on the banner "test system". Must be `false` for production server.
-ckanext.odsh.language_mapping | `string` | Absolute path to language mapping file, e.g., `/usr/lib/ckan/default/src/ckanext-odsh/language_mapping.json`.
-ckanext.odsh.subject_mapping | `string` | Absolute path to subject mapping file, e.g., `/usr/lib/ckan/default/src/ckanext-odsh/subject_mapping.json`.
-ckanext.odsh.spatial.mapping | `string` | Absolute path to spatial mapping file, e.g., `/usr/lib/ckan/default/src/odsh-odsh/schleswig-holstein_geojson.csv`.
-ckanext.odsh.resource_formats_fallback_filepath | `string` | Absolute path to resource formats fallback file, e.g., `/usr/lib/ckan/default/src/ckanext-odsh/fileformats.rdf`
-ckanext.odsh.upload_formats | `string` | E.g., `pdf`.
-ckanext.odsh.require_at_least_one_category | `boolean` | E.g., `True`.
-ckanext.odsh.require_spatial_uri | `boolean` | E.g., `True`.
-ckanext.odsh.require_subject | `boolean` | E.g., `False`.
-ckanext.odsh.is_optional_temporal_start | `boolean` | E.g., `False`.
+Parameter | Type | Default | Description
+--------- | ---- | ------- | -----------
+`ckanext.odsh.home` | `string` | `/usr/lib/ckan/default/src/ckanext-odsh` | Absolute path to base directory.
+`ckanext.odsh.public_url` | `string` | - | The public URL, e.g., `https://opendata.schleswig-holstein.de`.
+`ckanext.odsh.showtestbanner` | `boolean` | `True` | Switches on the banner "test system". Must be `false` for production server.
+`ckanext.odsh.language_mapping` | `string` | `/usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/resources/language_mapping.json` | Absolute path to language mapping file.
+`ckanext.odsh.subject_mapping` | `string` | `/usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/resources/subject_mapping.json` | Absolute path to subject mapping file.
+`ckanext.odsh.spatial.mapping` | `string` | `/usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/resources/schleswig-holstein_geojson.csv` | Absolute path to spatial mapping file. The mapping file is expected to be a tab-separated file with three columns: URI, spatial text, and JSON geometry.
+`ckanext.odsh.resource_formats_fallback_filepath` | `string` | `/usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/resources/fileformats.rdf` | Absolute path to resource formats fallback file.
+`ckanext.odsh.upload_formats` | `string` | - | Known upload formats, e.g., `pdf`.
+`ckanext.odsh.require_at_least_one_category` | `boolean` | `False` | TODO
+`ckanext.odsh.require_spatial_uri` | `boolean` | `False` | Indicates whether a spatial URI is required for the dataset.
+`ckanext.odsh.require_subject` | `boolean` | `True` | Indicates whether a subject is required for a dataset.
+`ckanext.odsh.is_optional_temporal_start` | `boolean` | E.g., `False`.
+`ckanext.odsh.download_proxy` | `string` | Use proxy server to access the web, e.g., `http://1.2.3.4:4123`.
+`copy_remote_resources` | `boolean` | `False` | Indicates whether remote resources should be copied when creating a resource.
+`ckanext.odsh.lenient_with` | | |
+`ckanext.odsh.testuser` | `string` | `None` | Name of user for testing.
+`ckanext.odsh.testuserpass` | `string` | `None` | Password of user for testing.
### Testing
diff --git a/ckanext/odsh/helpers.py b/ckanext/odsh/helpers.py
index 785e9c2d..ddcea24e 100644
--- a/ckanext/odsh/helpers.py
+++ b/ckanext/odsh/helpers.py
@@ -25,6 +25,7 @@ import subprocess
import ckan.lib.helpers as helpers
import os.path
from collections import OrderedDict
+import pkg_resources
get_action = logic.get_action
log = logging.getLogger(__name__)
@@ -226,10 +227,11 @@ def odsh_public_url():
return config.get('ckanext.odsh.public_url')
def spatial_extends_available():
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ file_path = config.get('ckanext.odsh.spatial.mapping', extension_path + '/resources/schleswig-holstein_geojson.csv')
- mapping_file = config.get('ckanext.odsh.spatial.mapping')
try:
- mapping_file = urllib.request.urlopen(mapping_file)
+ mapping_file = urllib.request.urlopen(file_path)
except Exception:
raise Exception("Could not load spatial mapping file!")
@@ -253,7 +255,7 @@ def odsh_public_resource_url(res):
def odsh_get_version_id():
try:
- home = config.get('ckanext.odsh.home', None)
+ home = config.get('ckanext.odsh.home', '/usr/lib/ckan/default/src/ckanext-odsh')
if home:
if home[-1] == '/':
home = home[:-1]
@@ -265,7 +267,7 @@ def odsh_get_version_id():
def odsh_show_testbanner():
- return config.get('ckanext.odsh.showtestbanner', 'False') == 'True'
+ return config.get('ckanext.odsh.showtestbanner', False) == True
def odsh_get_facet_items_dict(name, limit=None):
@@ -457,8 +459,8 @@ def odsh_load_mdk_sample_dataset():
and corresponding mapping in mdk_mapping.json file.
'''
- path = os.path.abspath(os.path.dirname(__file__))
- default_sample_data_file_path = os.path.join(path, "../../mdk_mapping.json")
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ default_sample_data_file_path = extension_path + '/resources/mdk_mapping.json'
sample_data_file_path = config.get(
'ckanext.odsh.sample_data_file_path', default_sample_data_file_path)
@@ -490,8 +492,8 @@ def odsh_load_raw_mdk_sample_dataset():
and corresponding mapping in mdk_mapping.json file.
'''
- path = os.path.abspath(os.path.dirname(__file__))
- default_sample_data_file_path = os.path.join(path, "../../mdk_mapping.json")
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ default_sample_data_file_path = extension_path + '/resources/mdk_mapping.json'
sample_data_file_path = config.get(
'ckanext.odsh.sample_data_file_path', default_sample_data_file_path)
diff --git a/ckanext/odsh/helpers_tpsh.py b/ckanext/odsh/helpers_tpsh.py
index e543c5cc..c7541af5 100644
--- a/ckanext/odsh/helpers_tpsh.py
+++ b/ckanext/odsh/helpers_tpsh.py
@@ -9,6 +9,7 @@ import urllib.request, urllib.error, urllib.parse
from collections import OrderedDict
import subprocess
import os
+import pkg_resources
from ckan.common import config
import ckan.lib.helpers as helpers
@@ -82,7 +83,9 @@ def get_pkg_relationships_from_model(pkg_dict):
return model.Package.get(pkg_id).get_relationships()
def load_language_mapping():
- with open(config.get('ckanext.odsh.language_mapping')) as language_mapping_json:
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ file_path = config.get('ckanext.odsh.language_mapping', extension_path + '/resources/language_mapping.json')
+ with open(file_path) as language_mapping_json:
LANGUAGE_MAPPING = json.loads(language_mapping_json.read())
return LANGUAGE_MAPPING
@@ -90,7 +93,9 @@ def load_json_to_ordered_dict(json_str):
return json.loads(json_str, object_pairs_hook=OrderedDict)
def load_subject_mapping():
- with open(config.get('ckanext.odsh.subject_mapping')) as subject_mapping_json:
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ file_path = config.get('ckanext.odsh.subject_mapping', extension_path + '/resources/subject_mapping.json')
+ with open(file_path) as subject_mapping_json:
SUBJECT_MAPPING = load_json_to_ordered_dict(subject_mapping_json.read())
return SUBJECT_MAPPING
@@ -125,8 +130,9 @@ def _get_language_id(pkg_dict):
return language_id_cleaned
def get_spatial_for_selection():
- mapping_path = config.get('ckanext.odsh.spatial.mapping')
- with open(mapping_path, newline='') as mapping_file:
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ file_path = config.get('ckanext.odsh.spatial.mapping', extension_path + '/resources/schleswig-holstein_geojson.csv')
+ with open(file_path, newline='') as mapping_file:
cr = csv.reader(mapping_file, delimiter="\t")
spatial_mapping = list()
for row in cr:
diff --git a/ckanext/odsh/logic/action.py b/ckanext/odsh/logic/action.py
index bf1333c3..2985fb2f 100644
--- a/ckanext/odsh/logic/action.py
+++ b/ckanext/odsh/logic/action.py
@@ -156,7 +156,7 @@ def autocomplete(context, data_dict):
def odsh_resource_create(context, data_dict):
copy_remote_resources = toolkit.asbool(
- tk.config.get('ckanext.odsh.copy_remote_resources', 'False')
+ tk.config.get('ckanext.odsh.copy_remote_resources', False)
)
if copy_remote_resources:
is_linked_resource = ( not 'upload' in data_dict ) or ( not isinstance(data_dict['upload'], cgi.FieldStorage))
diff --git a/ckanext/odsh/precondition.py b/ckanext/odsh/precondition.py
deleted file mode 100644
index 5314133f..00000000
--- a/ckanext/odsh/precondition.py
+++ /dev/null
@@ -1,5 +0,0 @@
-import ckan.plugins.toolkit as tk
-
-class PreconditionViolated(Exception):
- def __init__(self, message):
- super(PreconditionViolated, self).__init__(message)
diff --git a/ckanext/odsh/profiles/odsh_european_dcatap_profile.py b/ckanext/odsh/profiles/odsh_european_dcatap_profile.py
index 4b56438b..2f73bd09 100644
--- a/ckanext/odsh/profiles/odsh_european_dcatap_profile.py
+++ b/ckanext/odsh/profiles/odsh_european_dcatap_profile.py
@@ -5,6 +5,7 @@ from ckan.common import config, json
from ckan.model.license import LicenseRegister
from ckanext.dcat.profiles import EuropeanDCATAPProfile, DCT, URIRefOrLiteral
from ckanext.dcatde.profiles import DCAT
+import pkg_resources
log = logging.getLogger(__name__)
DCT = rdflib.namespace.Namespace("http://purl.org/dc/terms/")
@@ -77,8 +78,9 @@ def resource_formats():
g = rdflib.Graph()
# Something went wrong with trying to get the file formats online, try to use backup instead
try:
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
fallback_filepath = config.get(
- 'ckanext.odsh.resource_formats_fallback_filepath')
+ 'ckanext.odsh.resource_formats_fallback_filepath', extension_path + '/resources/fileformats.rdf')
g.parse(fallback_filepath)
assert len(set([s for s in g.subjects()])) > 120
except:
@@ -120,11 +122,12 @@ def get_language():
global _LANGUAGES
if not _LANGUAGES:
_LANGUAGES = {}
- languages_file_path = config.get('ckanext.odsh.language.mapping')
+ languages_file_path = config.get('ckanext.odsh.language_mapping')
if not languages_file_path:
log.warning(
- "Could not find config setting: 'ckanext.odsh.language.mapping', using fallback instead.")
- languages_file_path = '/usr/lib/ckan/default/src/ckanext-odsh/languages.json'
+ "Could not find config setting: 'ckanext.odsh.language_mapping', using fallback instead.")
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ languages_file_path = extension_path + '/resources/languages.json'
with open(languages_file_path) as languages_file:
try:
language_mapping_table = json.loads(languages_file.read())
diff --git a/fileformats.rdf b/ckanext/odsh/resources/fileformats.rdf
similarity index 100%
rename from fileformats.rdf
rename to ckanext/odsh/resources/fileformats.rdf
diff --git a/language_mapping.json b/ckanext/odsh/resources/language_mapping.json
similarity index 100%
rename from language_mapping.json
rename to ckanext/odsh/resources/language_mapping.json
diff --git a/languages.json b/ckanext/odsh/resources/languages.json
similarity index 100%
rename from languages.json
rename to ckanext/odsh/resources/languages.json
diff --git a/licenses.json b/ckanext/odsh/resources/licenses.json
similarity index 100%
rename from licenses.json
rename to ckanext/odsh/resources/licenses.json
diff --git a/mdk_mapping.json b/ckanext/odsh/resources/mdk_mapping.json
similarity index 100%
rename from mdk_mapping.json
rename to ckanext/odsh/resources/mdk_mapping.json
diff --git a/ckanext/odsh/resource_format_openness_scores.json b/ckanext/odsh/resources/resource_format_openness_scores.json
similarity index 100%
rename from ckanext/odsh/resource_format_openness_scores.json
rename to ckanext/odsh/resources/resource_format_openness_scores.json
diff --git a/schleswig-holstein_geojson.csv b/ckanext/odsh/resources/schleswig-holstein_geojson.csv
similarity index 100%
rename from schleswig-holstein_geojson.csv
rename to ckanext/odsh/resources/schleswig-holstein_geojson.csv
diff --git a/subject_mapping.json b/ckanext/odsh/resources/subject_mapping.json
similarity index 100%
rename from subject_mapping.json
rename to ckanext/odsh/resources/subject_mapping.json
diff --git a/ckanext/odsh/tests/test_env.py b/ckanext/odsh/tests/test_env.py
index 70e2bf12..9ad5a394 100644
--- a/ckanext/odsh/tests/test_env.py
+++ b/ckanext/odsh/tests/test_env.py
@@ -115,8 +115,8 @@ class TestEnv:
checkJsonFile(
'qa.resource_format_openness_scores_json', expectedLength=60)
- checkConfig('ckanext.odsh.language.mapping',
- '/usr/lib/ckan/default/src/ckanext-odsh/languages.json')
+ checkConfig('ckanext.odsh.language_mapping',
+ '/usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/resources/languages.json')
def test_non_critical(self):
checkConfig('who.timeout', '1800')
diff --git a/ckanext/odsh/tests/test_helpers.py b/ckanext/odsh/tests/test_helpers.py
index f033b573..33f03d58 100644
--- a/ckanext/odsh/tests/test_helpers.py
+++ b/ckanext/odsh/tests/test_helpers.py
@@ -26,7 +26,7 @@ def change_config():
def changed_config():
_original_config = config.copy()
config['ckanext.odsh.spatial.mapping'] = 'file:///usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/tests/spatial_mapping.csv'
- config['licenses_group_url'] = 'file:///usr/lib/ckan/default/src/ckanext-odsh/licenses.json'
+ config['licenses_group_url'] = 'file:///usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/ckanext/odsh/resources/licenses.json'
try:
yield
finally:
diff --git a/ckanext/odsh/tests_tpsh/test_helpers_tpsh.py b/ckanext/odsh/tests_tpsh/test_helpers_tpsh.py
index dc554cef..aeb3d60d 100644
--- a/ckanext/odsh/tests_tpsh/test_helpers_tpsh.py
+++ b/ckanext/odsh/tests_tpsh/test_helpers_tpsh.py
@@ -161,7 +161,7 @@ class Test_correct_missing_relationship(unittest.TestCase):
class Test_get_language_of_package(unittest.TestCase):
def setUp(self):
- config.update({'ckanext.odsh.language_mapping': '/usr/lib/ckan/default/src/ckanext-odsh/language_mapping.json'})
+ config.update({'ckanext.odsh.language_mapping': '/usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/resources/language_mapping.json'})
def tearDown(self):
config.clear()
diff --git a/ckanext/odsh/validation.py b/ckanext/odsh/validation.py
index a27fbf9c..4185d893 100644
--- a/ckanext/odsh/validation.py
+++ b/ckanext/odsh/validation.py
@@ -13,6 +13,7 @@ from ckan.lib.navl.dictization_functions import Missing
from ckanext.odsh.helpers_tpsh import get_package_dict
import ckan.plugins.toolkit as tk
+import pkg_resources
_ = toolkit._
@@ -218,7 +219,8 @@ def known_spatial_uri(key, data, errors, context):
data[('extras', new_index+1, 'value')] = poly
return
- mapping_path = tk.config.get('ckanext.odsh.spatial.mapping')
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
+ mapping_path = tk.config.get('ckanext.odsh.spatial.mapping', extension_path + '/resources/schleswig-holstein_geojson.csv')
not_found = True
spatial_text = str()
@@ -304,9 +306,9 @@ def _convert_subjectID_to_subjectText(subject_id, flattened_data):
if not subject_id:
return flattened_data
- default_subject_mapping_file_path = '/usr/lib/ckan/default/src/ckanext-odsh/subject_mapping.json'
+ extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
subject_mapping_file_path = tk.config.get(
- 'ckanext.odsh.subject_mapping_file_path', default_subject_mapping_file_path)
+ 'ckanext.odsh.subject_mapping_file_path', extension_path + '/resources/subject_mapping.json')
try:
with open(subject_mapping_file_path) as mapping_json:
--
GitLab