From b9634780d18f4751be166b96a4f32bc4010987b1 Mon Sep 17 00:00:00 2001
From: Rainer Herzog <herzog@informatik.uni-hamburg.de>
Date: Fri, 25 Jan 2019 03:34:28 +0100
Subject: [PATCH] Also accept datasets from Statistikamt Nord without
temporal_start/end
---
.../harvesters/statistikamtnordharvester.py | 32 +++++++------------
ckanext/odsh/plugin.py | 19 ++++++-----
2 files changed, 22 insertions(+), 29 deletions(-)
diff --git a/ckanext/odsh/harvesters/statistikamtnordharvester.py b/ckanext/odsh/harvesters/statistikamtnordharvester.py
index d38d01ca..aea0a1ea 100755
--- a/ckanext/odsh/harvesters/statistikamtnordharvester.py
+++ b/ckanext/odsh/harvesters/statistikamtnordharvester.py
@@ -125,19 +125,6 @@ class StatistikamtNordHarvester(ODSHBaseHarvester):
self.map_fields(context, harvest_object)
return True
- @staticmethod
- def _update_schema(schema):
- schema.update({'temporal_start': [
- toolkit.get_validator('ignore_empty'),
- toolkit.get_converter('convert_to_extras')]})
- schema.update({'temporal_end': [
- toolkit.get_validator('ignore_empty'),
- toolkit.get_converter('convert_to_extras')]})
- schema.update({'issued': [
- toolkit.get_validator('ignore_missing'),
- toolkit.get_validator('ignore_empty'),
- toolkit.get_converter('convert_to_extras')]})
-
def map_fields(self, context, harvest_object):
values = json.loads(harvest_object.content)
@@ -176,9 +163,7 @@ class StatistikamtNordHarvester(ODSHBaseHarvester):
try:
context = {'user': self._get_user_name(), 'return_id_only': True, 'ignore_auth': True}
package_plugin = lib_plugins.lookup_package_plugin(package_dict.get('type', None))
- package_schema = package_plugin.create_package_schema()
- self._update_schema(package_schema)
- context['schema'] = package_schema
+ context['schema'] = package_plugin.create_package_schema()
self._handle_current_harvest_object(harvest_object, harvest_object.guid)
result = toolkit.get_action('package_create')(context, package_dict)
return result
@@ -190,11 +175,16 @@ class StatistikamtNordHarvester(ODSHBaseHarvester):
def add_extras(package_dict, values):
# issued sollte noch geliefert werden!
package_dict['extras'].append({
- 'key': 'issued', 'value': datetime.datetime.now().isoformat().split('T')[0]})
- package_dict['extras'].append({
- 'key': 'temporal_start', 'value': values['ZeitraumVon']})
- package_dict['extras'].append({
- 'key': 'temporal_end', 'value': values['ZeitraumBis']})
+ 'key': 'issued', 'value': datetime.datetime.now().isoformat()})
+ try:
+ if values['ZeitraumVon'] != "":
+ package_dict['extras'].append({
+ 'key': 'temporal_start', 'value': values['ZeitraumVon']})
+ if values['ZeitraumBis'] != "":
+ package_dict['extras'].append({
+ 'key': 'temporal_end', 'value': values['ZeitraumBis']})
+ except KeyError as kerr:
+ log.debug("Date not available: " + str(kerr))
package_dict['extras'].append({
'key': 'spatial_uri', 'value': 'http://dcat-ap.de/def/politicalGeocoding/stateKey/01'})
package_dict['extras'].append({
diff --git a/ckanext/odsh/plugin.py b/ckanext/odsh/plugin.py
index 7f79c619..0edcd9eb 100644
--- a/ckanext/odsh/plugin.py
+++ b/ckanext/odsh/plugin.py
@@ -129,14 +129,17 @@ def odsh_validate_extra_date(key, field, data, errors, context):
value = _extract_value(key, data, field)
if not value:
- raise toolkit.Invalid(field+':odsh_'+field+'_error_label')
-
- try:
- # date.split('T')[0] will yield "2012-01-01"
- # no matter if the date is like "2012-01-01" or "2012-01-01T00:00:00"
- datetime.datetime.strptime(value.split('T')[0], '%Y-%m-%d')
- except ValueError:
- raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label')
+ # Statistikamt Nord does not always provide temporal_start/end,
+ # but their datasets have to be accepted as they are.
+ if data[('id',)][:7] != 'StaNord':
+ raise toolkit.Invalid(field+':odsh_'+field+'_error_label')
+ else:
+ try:
+ # date.split('T')[0] will yield "2012-01-01"
+ # no matter if the date is like "2012-01-01" or "2012-01-01T00:00:00"
+ datetime.datetime.strptime(value.split('T')[0],'%Y-%m-%d').isoformat()
+ except ValueError:
+ raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label')
def odsh_validate_extra_date_factory(field):
--
GitLab