Changed extra field handling in harvesters (extra fields are never converted to the top level)

ckanext/odsh/harvesters/kielharvester.py

@@ -124,16 +124,20 @@ class KielHarvester(ODSHBaseHarvester):
             extras = package_dict['extras']
             new_extras = list()
             for extra in extras:
-                if extra['key'] in ['temporal_start', 'temporal_end', 'issued']:
+                # WARNING: When this code was written, all datasets had '-zero-' licences, i.e.
+                # there was no key 'licenseAttributionByText' which we would expect for '-by-' licences.
+                # The setting is just anticipated, matching for datasets with a corresponding licence.
+                if extra['key'] == 'licenseAttributionByText':
+                    new_extras.append(extra)
+                elif extra['key'] in ['temporal_start', 'temporal_end', 'issued']:
                     new_extras.append(extra)
 
             new_extras.append(
-                {'spatial_uri': 'http://dcat-ap.de/def/politicalGeocoding/districtKey/01002'})
+                {'key': 'spatial_uri',
+                 'value': 'http://dcat-ap.de/def/politicalGeocoding/districtKey/01002'})
 
             package_dict['extras'] = new_extras
 
-            log.debug(package_dict['extras'])
-
             license_id = self._get_license_id(package_dict['license_id'])
             if license_id:
                 package_dict['license_id'] = license_id

ckanext/odsh/harvesters/statistikamtnordharvester.py

@@ -142,7 +142,7 @@ class StatistikamtNordHarvester(ODSHBaseHarvester):
         values = json.loads(harvest_object.content)
 
         package_dict = dict()
-        package_dict.update({'resources': [], 'tags': [], 'groups': []})
+        package_dict.update({'resources': [], 'tags': [], 'groups': [], 'extras': []})
         title = values['Titel']
         package_dict.update({'title': title})
         package_dict.update({'name': self._gen_new_name(title)})
@@ -161,17 +161,10 @@ class StatistikamtNordHarvester(ODSHBaseHarvester):
             package_dict['url'] = ""
         package_dict.update({'type': 'dataset'})
 
-        package_dict.update({'licenseAttributionByText': 'Statistisches Amt für Hamburg und Schleswig-Holstein -'
-                                                         ' Anstalt des öffentlichen Rechts - (Statistikamt Nord)'})
-        package_dict.update({'temporal_start': values['ZeitraumVon']})
-        package_dict.update({'temporal_end': values['ZeitraumBis']})
-        package_dict.update({'spatial_uri': 'http://dcat-ap.de/def/politicalGeocoding/stateKey/01'})
-        # issued sollte noch geliefert werden!
-        package_dict.update({'issued': datetime.datetime.now()})
         self.add_ressources(package_dict, values)
 
         self.add_tags(package_dict, values)
-
+        self.add_extras(package_dict, values)
         self.map_to_group(package_dict, values)
 
         source_dataset = get_action('package_show')(context.copy(), {'id': harvest_object.source.id})
@@ -193,6 +186,22 @@ class StatistikamtNordHarvester(ODSHBaseHarvester):
             self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import')
             return False
 
+    @staticmethod
+    def add_extras(package_dict, values):
+        # issued sollte noch geliefert werden!
+        package_dict['extras'].append({
+          'key': 'issued', 'value': datetime.datetime.now().isoformat().split('T')[0]})
+        package_dict['extras'].append({
+          'key': 'temporal_start', 'value': values['ZeitraumVon']})
+        package_dict['extras'].append({
+          'key': 'temporal_end', 'value': values['ZeitraumBis']})
+        package_dict['extras'].append({
+          'key': 'spatial_uri', 'value': 'http://dcat-ap.de/def/politicalGeocoding/stateKey/01'})
+        package_dict['extras'].append({
+          'key': 'licenseAttributionByText',
+          'value': 'Statistisches Amt für Hamburg und Schleswig-Holstein - '
+              'Anstalt des öffentlichen Rechts - (Statistikamt Nord)'})
+
     @staticmethod
     def add_tags(package_dict, values):
         tags = values['Schlagwoerter']['Schlagwort']

ckanext/odsh/plugin.py

@@ -132,7 +132,9 @@ def odsh_validate_extra_date(key, field, data, errors, context):
         raise toolkit.Invalid(field+':odsh_'+field+'_error_label')
 
     try:
-        datetime.datetime.strptime(value, '%Y-%m-%d')
+        # date.split('T')[0] will yield "2012-01-01"
+        # no matter if the date is like "2012-01-01" or "2012-01-01T00:00:00"
+        datetime.datetime.strptime(value.split('T')[0], '%Y-%m-%d')
     except ValueError:
         raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label')