Transparenzportal v1.1 -> v1.3

Änderungen die zwischen den Versionen 1.1 und 1.3 des Transparenzportals vorgenommen wurden, in den Code für das Open-Data-Portal übernommen.

ckanext/odsh/collection/controller.py

 from ckan.lib.helpers import is_url, url_for
 import ckan.plugins.toolkit as toolkit
 from ckan.controllers.package import PackageController
-from helpers import get_latest_resources_for_type, get_latest_dataset
+from helpers import get_latest_resources_for_format, get_latest_dataset
 
 
 
@@ -13,10 +13,10 @@ class LatestDatasetController(PackageController):
 
 class LatestRecourcesController(PackageController):
     
-    def latest_resource(self, id, type):
-        latest_resources = get_latest_resources_for_type(id, type)
+    def latest_resource(self, id, resource_format):
+        latest_resources = get_latest_resources_for_format(id, resource_format)
         if latest_resources is None:
-            abort(404)
+            toolkit.abort(404)
         url_type = latest_resources.get('url_type')
         if url_type is None:
             resource_url = latest_resources.get('url')
@@ -35,4 +35,4 @@ class LatestRecourcesController(PackageController):
                                     filename=pre_resource_url,
                                     qualified = True)
             toolkit.redirect_to(url_resource)
-        abort(404)
+        toolkit.abort(404)
\ No newline at end of file

ckanext/odsh/collection/helpers.py

 from string import lower
+from operator import itemgetter
 
 import ckan.lib.helpers as helpers
 import ckan.model as model
 import ckan.plugins.toolkit as toolkit
 
 
+def get_collection(dataset_dict):
+    collection_id = get_collection_id(dataset_dict)
+    if collection_id:
+        return get_collection_info(collection_id, dataset_dict)
+    
+    return None
+
+
+def get_collection_info(collection_id, dataset_dict=None):
+    collection_dict = get_package_dict(collection_id)
+    dataset_names = get_dataset_names(collection_dict)
+    datasets_in_collection = get_datasets_from_solr(dataset_names)
+    collection_info = gather_collection_info(collection_dict, datasets_in_collection, dataset_dict)
+    return collection_info
+
+
+def get_collection_id(dataset_dict):
+    relationships_dataset = dataset_dict.get('relationships_as_subject')
+    if relationships_dataset and len(relationships_dataset):        
+        return relationships_dataset[0]['__extras']['object_package_id']
+    relationships_dataset = dataset_dict.get('relationships')
+    if relationships_dataset and len(relationships_dataset):
+        return relationships_dataset[0].get('object')
+    return None
 
-#routine functions
 
 def get_package_dict(name):
     return model.Package.get(name).as_dict()
 
-def get_relationships(name):
-    collection_dict = get_package_dict(name)
-    return collection_dict.get('relationships')
 
-def get_all_datasets_belonging_to_collection(collection_name):
-    list_rel_collection = get_relationships(collection_name)
-    name_list = list()
-    for item in list_rel_collection:
-        item_object = item.get('object') 
-        name_list.append(item_object)
-    return name_list
+def get_dataset_names(collection_dict):
+    collection_dict = get_package_dict(collection_dict.get('id')) # needed to get full package_dict
+    relationships_collection = collection_dict.get('relationships')
+    names_collection_members = [relationship.get('object') for relationship in relationships_collection]
+    return names_collection_members
 
-#for mapping latest resources and latest dataset
 
-def get_latest_dataset(collection_name):
-    collection_list_relationships = get_all_datasets_belonging_to_collection(collection_name)
-    latest_issued = latest_name = None
-    for item in collection_list_relationships:
-        item_pkt_dict =  get_package_dict(item)
-        if helpers.check_access('package_show', item_pkt_dict):
-            item_issued = item_pkt_dict.get('extras').get('issued')
-            if latest_issued < item_issued or (latest_issued == item_issued and latest_name < item):
-                latest_name=item
-                latest_issued=item_issued
-    return latest_name
+def get_datasets_from_solr(dataset_names):
+    context = None
 
+    name_expression = ' OR '.join(dataset_names)
+    fq = 'name:({})'.format(name_expression)
     
-def is_latest_resources(resource_format, type, resource_created,latest_created, resource_id, latest_id):
-    if lower(resource_format) == lower(type):
-        return (resource_created > latest_created or (resource_created == latest_created and resource_id > latest_id))
-    else:
-        return False
-
-def get_latest_resources_for_type(collection_name, type):
-    latest_dataset_name = get_latest_dataset(collection_name)
-    latest_dataset = get_package_dict(latest_dataset_name)
-    resource_list = latest_dataset.get('resources')
-    latest_resource = latest_created = latest_id = None
-    for resource in resource_list:
-        resource_format = resource.get('format')
-        resource_created = resource.get('created')
-        resource_id = resource.get('id')
-        if is_latest_resources(resource_format, type, resource_created, latest_created, resource_id, latest_id):
-            latest_id=resource_id
-            latest_created=resource_created
-            latest_resource=resource
-    return latest_resource
-
-#for predecessor and successor
-
-
-
-def get_collection_name_by_dataset(dataset_name):
-    list_rel_dataset = get_relationships(dataset_name)
-    if len(list_rel_dataset):        
-        return list_rel_dataset[0]['object']
-
-def get_collection_title_by_dataset(pkg_dict_dataset):
-    dataset_name = pkg_dict_dataset.get('name')
-    collection_name = get_collection_name_by_dataset(dataset_name)
-    if not collection_name:
+    sort = 'name asc,extras_issued asc'
+    
+    # maximum possible number of results is 1000, 
+    # see https://docs.ckan.org/en/ckan-2.7.3/api/index.html#ckan.logic.action.get.package_search
+    query_result = toolkit.get_action('package_search')(context, {
+        'fq': fq,
+        'sort': sort,
+        'rows': 1000, 
+    })
+
+    results = query_result.get('results')
+    datasets_found = results if results else []
+
+    return datasets_found
+
+
+def gather_collection_info(collection_dict, datasets_in_collection, dataset_dict=None):
+    name_first_dataset = datasets_in_collection[0].get('name')
+    url_first_dataset = url_from_id(name_first_dataset)
+    
+    name_last_dataset = datasets_in_collection[-1].get('name')
+    url_last_dataset = url_from_id(name_last_dataset)
+
+    name_collection = collection_dict.get('name')
+    persistent_link_last_member = url_last_member(name_collection)
+
+    url_collection = url_from_id(collection_dict.get('id'))
+
+    if dataset_dict:
+        name_current_dataset = dataset_dict.get('name')
+        dataset_names = [d.get('name') for d in datasets_in_collection]
+        
+        def get_predecessor():
+            try:
+                id_current = dataset_names.index(name_current_dataset)
+            except ValueError:
                 return None
-    context = None
-    pkg_dict_collection = toolkit.get_action('package_show')(context, {'id': collection_name})
-    if not pkg_dict_collection:
+            if id_current > 0:
+                return dataset_names[id_current - 1]
+            return None
+        
+        def get_successor():
+            try:
+                id_current = dataset_names.index(name_current_dataset)
+            except ValueError:
                 return None
-    title_collection = pkg_dict_collection.get('title')
-    return title_collection
-
-def get_all_datasets_belonging_to_collection_by_dataset(dataset_name):
-    collection_name = get_collection_name_by_dataset(dataset_name)
-    if collection_name: 
-        return get_all_datasets_belonging_to_collection(collection_name)
-    return []
-
-def _get_siblings_dicts_with_access(pkg_dict):
-    dataset_name = pkg_dict.get('name')
-    list_of_siblings = get_all_datasets_belonging_to_collection_by_dataset(dataset_name)
-    n_siblings = len(list_of_siblings)
-    if n_siblings>0:
-        siblings_dicts = [get_package_dict(name) for name in list_of_siblings]
-        user_has_access = lambda pkg_dict:helpers.check_access('package_show', pkg_dict)
-        siblings_dicts_with_access = filter(user_has_access, siblings_dicts)
-        return siblings_dicts_with_access
+            if id_current < len(dataset_names) - 1:
+                return dataset_names[id_current + 1]
             return None
         
-def _sort_siblings_by_name_and_date(siblings_dicts):
-    '''
-    sort by name first and then by date to have a fallback if dates are the same
-    '''
-    _get_name = lambda pkg_dict:pkg_dict.get('name')
-    _get_issued = lambda pkg_dict:pkg_dict.get('extras').get('issued')
-    siblings_dicts_sorted_by_name = sorted(siblings_dicts, key=_get_name)
-    siblings_dicts_sorted_by_date_issued = sorted(siblings_dicts_sorted_by_name, key=_get_issued)
-    return siblings_dicts_sorted_by_date_issued
-
-def get_successor_and_predecessor_dataset(pkg_dict):
-    dataset_name = pkg_dict.get('name')
-    siblings_dicts_with_access = _get_siblings_dicts_with_access(pkg_dict)
-    if siblings_dicts_with_access:
-        n_siblings = len(siblings_dicts_with_access)
-        siblings_dicts_sorted_by_date_issued = _sort_siblings_by_name_and_date(siblings_dicts_with_access)
-        siblings_names_sorted_by_date_issued = [d['name'] for d in siblings_dicts_sorted_by_date_issued]
-        id_current_dataset = siblings_names_sorted_by_date_issued.index(dataset_name)
-        predecessor_name = (
-            siblings_names_sorted_by_date_issued[id_current_dataset-1] if (id_current_dataset > 0) 
-            else None
-        )
-        successor_name = (
-            siblings_names_sorted_by_date_issued[id_current_dataset+1] if (id_current_dataset < n_siblings-1) 
-            else None
-        )
+        name_predecessor = get_predecessor()
+        url_predecessor = url_from_id(name_predecessor) if name_predecessor else None
+        
+        name_successor = get_successor()
+        url_successor = url_from_id(name_successor) if name_successor else None
     else:
-        predecessor_name, successor_name = None, None
-    return successor_name, predecessor_name
-
-def get_successor_and_predecessor_urls(pkg_dict):
-    successor_name, predecessor_name = get_successor_and_predecessor_dataset(pkg_dict)
-    successor_url, predecessor_url = (
-        helpers.url_for(controller='package', action='read', id=name)
-        if name is not None
-        else None
-        for name in (successor_name, predecessor_name)
+        url_predecessor = url_successor = None
+    
+    return {
+        'title': collection_dict.get('title'),
+        'url': url_collection,
+        'members': datasets_in_collection,
+        'first_member': {
+            'name': name_first_dataset,
+            'url': url_first_dataset,
+        },
+        'last_member': {
+            'name': name_last_dataset,
+            'url': url_last_dataset,
+        },
+        'predecessor': {
+            'url': url_predecessor,
+        },
+        'successor': {
+            'url': url_successor,
+        },
+        'persistent_link_last_member': persistent_link_last_member,
+    }
+
+def url_from_id(package_id):
+    return helpers.url_for(controller='package', action='read', id=package_id)
+
+def url_last_member(name_collection):
+    return helpers.url_for(
+        controller='ckanext.odsh.collection.controller:LatestDatasetController', 
+        action='latest',
+        id=name_collection
     )
-    return successor_url, predecessor_url
 
-def get_successor(pkg_dict):
-    successor_and_predecessor = get_successor_and_predecessor_urls(pkg_dict)
-    return successor_and_predecessor[0]
 
-def get_predecessor(pkg_dict):
-    successor_and_predecessor = get_successor_and_predecessor_urls(pkg_dict)
-    return successor_and_predecessor[1]
+def get_latest_dataset(collection_name):
+    collection_info = get_collection_info(collection_name)
+    latest_name = collection_info['last_member']['name']
+    return latest_name
 
-#link to latest collection member
-def latest_collection_member_persistent_link(pkg_dict):
-    dataset_name = pkg_dict.get('name')
-    collection_name = get_collection_name_by_dataset(
-        dataset_name=dataset_name
-    )
-    if not collection_name:
+
+def get_latest_resources_for_format(collection_name, resource_format):
+    collection_info = get_collection_info(collection_name)
+    members = collection_info.get('members')
+    if not members:
         return None
-    url = helpers.url_for(
-        controller='ckanext.odsh.collection.controller:LatestDatasetController', 
-        action='latest',
-        id=collection_name
-    )
-    return url
+    latest_dataset = members[-1]
+    resources = latest_dataset.get('resources')
+    if not resources:
+        return None
+    resources_with_asked_type = [r for r in resources if r.get('format').upper() == resource_format.upper()]
+    resources_sorted = sorted(resources_with_asked_type, key=itemgetter('id','created'), reverse=True)
\ No newline at end of file
+    return resources_sorted[-1]

ckanext/odsh/collection/plugin.py

 
 from ckan.lib.plugins import DefaultTranslation, DefaultDatasetForm
 import ckan.plugins as plugins
+import ckan.plugins.toolkit as toolkit
 import helpers as collection_helpers
 from routes.mapper import SubMapper
 
@@ -17,6 +18,9 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
     def is_fallback(self):
         return False
     
+    def read_template(self):
+        return 'package/collection_read.html'
+
     
     # IRoutes    
     def before_map(self, map):
@@ -28,7 +32,7 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
         )
 
         map.connect(
-            '/collection/{id}/aktuell.{type}',
+            '/collection/{id}/aktuell.{resource_format}',
             controller='ckanext.odsh.collection.controller:LatestRecourcesController',
             action='latest_resource'
         )
@@ -39,7 +43,7 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
             path_prefix='/collection/'
         ) as m:
             m.connect('latest', '{id}/aktuell', action='latest')
-            m.connect('latest_resource', '{id}/aktuell.{type}', action='latest_resource')  
+            m.connect('latest_resource', '{id}/aktuell.{resource_format}', action='latest_resource')  
         return map
 
     def after_map(self, map):
@@ -53,8 +57,8 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
         # other extensions.
 
         return {
-            'collection_get_successor': collection_helpers.get_successor,
-            'collection_get_predecessor': collection_helpers.get_predecessor,
-            'collection_get_latest_member':collection_helpers.latest_collection_member_persistent_link,
-            'collection_get_title': collection_helpers.get_collection_title_by_dataset,
+            'get_collection': collection_helpers.get_collection,
+            'get_collection_info': collection_helpers.get_collection_info,
+            'url_from_id': collection_helpers.url_from_id,
         }
+    
\ No newline at end of file

ckanext/odsh/controller.py

@@ -11,7 +11,6 @@ from ckan.controllers.package import PackageController
 from ckan.controllers.feed import FeedController, ITEMS_LIMIT, _package_search, _create_atom_id
 import ckan.lib.helpers as h
 import ckan.authz as authz
-from ckan.common import c
 import logging
 import matomo
 import ckan.logic as logic
@@ -21,6 +20,8 @@ import ckan.plugins.toolkit as toolkit
 from ckanext.dcat.controllers import DCATController
 import ckan.model as model
 import helpers
+import json
+
 
 abort = base.abort
 log = logging.getLogger(__name__)
@@ -30,10 +31,10 @@ get_action = logic.get_action
 
 class OdshRouteController(HomeController):
     def info_page(self):
-        h.redirect_to('http://www.schleswig-holstein.de/odpinfo')
+        h.redirect_to('http://www.schleswig-holstein.de/tpinfo')
 
     def start(self):
-        h.redirect_to('http://www.schleswig-holstein.de/odpstart')
+        h.redirect_to('http://www.schleswig-holstein.de/tpstart')
 
     def not_found(self):
         abort(404)
@@ -143,6 +144,12 @@ class OdshGroupController(OrganizationController):
 
 class OdshApiController(ApiController):
     def action(self, logic_function, ver=None):
+        if toolkit.asbool(config.get('ckanext.odsh.log_api_requests', 'false')):
+            try:
+                request_data = self._get_request_data(try_url_params=False)
+                log.info('POST request body: {}'.format(request_data))
+            except Exception, e:
+                log.error(e)
         if logic_function == 'resource_qv4yAI2rgotamXGk98gJ':
             return helpers.odsh_get_version_id()
         if logic_function == 'resourcelog_qv4yAI2rgotamXGk98gJ':

ckanext/odsh/helper_pkg_dict.py

@@ -102,18 +102,6 @@ class HelperPgkDict(object):
         return None
     
 
-    def get_collection_id(self):
-        '''
-        construct a collection uri from the id of 
-        the containing collection
-        '''
-        package_name = self.pkg_dict.get('name')
-        collection_name = helpers_collection.get_collection_name_by_dataset(package_name)
-        collection_dict = helpers_collection.get_package_dict(collection_name)
-        collection_id = collection_dict.get('id')
-        return collection_id
-
-    
     def add_uri_to_store(self):
         '''
         store pair of uri and id for subsequent use

ckanext/odsh/helpers_tpsh.py

@@ -8,6 +8,8 @@ import json
 import re
 import urllib2
 from collections import OrderedDict
+import subprocess
+import os
 
 from ckan.common import config
 import ckan.lib.helpers as helpers
@@ -170,7 +172,11 @@ def get_language_for_selection():
     return dict_for_select_box
 
 def get_package_dict(name):
-    return model.Package.get(name).as_dict()
+    '''
+    raises ckan.logic.NotFound if not found
+    '''
+    package_dict = toolkit.get_action('package_show')(None, {'id': name})
+    return package_dict
 
 def size_of_fmt(num, suffix='B'):
     for unit in ['',' k',' M',' G',' T',' P',' E',' Z']:
@@ -209,3 +215,11 @@ def get_body_mail(organization, package):
     mail_url = "URL: " +  url + "%0D%0A"  +  "%0D%0A" 
     message =  mail_titel + mail_document  +  mail_url + "Mein Kommentar:" +  "%0D%0A"    +  "%0D%0A"  +  "%0D%0A"  +  "%0D%0A" 
     return anrede + message
+
+def git_commit_hash():
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    try:
+        command = 'git log -n 1 --format=%H'
+    except:
+        return 'unknown'
+    return subprocess.check_output([command], shell=True, cwd=current_dir)
\ No newline at end of file

ckanext/odsh/logic/action.py

-import logging
 import ckan.logic as logic
 from ckan.logic.action.update import user_update
-from ckan.logic.action.create import package_create, user_create, group_member_create
+from ckan.logic.action.create import package_create, resource_create, user_create, group_member_create
 import ckan.model as model
 import ckan.lib.dictization.model_dictize as model_dictize
 from ckan.lib.munge import munge_title_to_name
 import ckan.plugins.toolkit as toolkit
-from ckan.lib.search.common import (
-    make_connection, SearchError, SearchQueryError
-)
+from ckan.lib.search.common import make_connection, SearchError
 import pysolr
 import datetime
+import cgi
+import urllib2
 
+import logging
 log = logging.getLogger(__name__)
 
+from ckanext.odsh.setup_proxy import setup_proxy, clear_proxy
+from ckanext.odsh.collection.helpers import get_collection_id, get_package_dict
+from ckanext.odsh.helpers_tpsh import add_pkg_to_collection
+from ckanext.odsh.helpers import odsh_extract_value_from_extras
+
 
 def odsh_package_create(context, data_dict):
     pkg_type = data_dict.get('type', None)
     if pkg_type == 'collection':
         return package_create(context, data_dict)
     munge_increment_name(data_dict)
+    related_package_name = data_dict.get('relatedPackage')
+    if related_package_name:
+        new_package = package_create(context, data_dict)
+        add_to_same_collection(related_package_name, new_package['name'])
+        return new_package
     if pkg_type != 'dataset':
         return package_create(context, data_dict)
-    issued = False
-    for extra in data_dict.get('extras'):
-        if extra['key'] == 'issued':
-            issued = True
-            break
-    if not issued:
-        data_dict['extras'].append({'key': 'issued', 'value': datetime.datetime.utcnow().isoformat()})
+    add_issued_if_missing(data_dict)
     return package_create(context, data_dict)
 
 
@@ -46,14 +50,53 @@ def munge_increment_name(data_dict):
         data_dict['name'] = name
 
     
+def add_issued_if_missing(data_dict):
+    issued = odsh_extract_value_from_extras(data_dict.get('extras'), 'issued')
+    if issued is None:
+        data_dict['extras'].append({'key': 'issued', 'value': datetime.datetime.utcnow().isoformat()})
+    
+
+def add_to_same_collection(related_package_name, package_name):
+    related_package = get_package_dict(related_package_name)
+    collection_id = get_collection_id(related_package)
+    if not collection_id:
+        collection_dict = auto_create_collection(related_package)
+        collection_id = collection_dict.get('id')
+        add_to_collection(collection_id, related_package_name)
+    add_to_collection(collection_id, package_name)
+
+
+def auto_create_collection(related_package):
+    related_package_title = related_package.get('title')
+    owner_org = related_package.get('owner_org')
+    _collection_dict = {
+        'title': related_package_title,
+        'owner_org': owner_org,
+        'type': 'collection',
+    }
+    munge_increment_name(_collection_dict)
+    collection_dict = package_create_via_toolkit(None, _collection_dict)
+    return collection_dict
+
+
+def package_create_via_toolkit(context, package_dict):
+    return toolkit.get_action('package_create')(context, package_dict)
+
+
+def add_to_collection(collection_id, package_name):
+    add_pkg_to_collection(package_name, collection_id)
+
+
 def check_password(password):
     return (len(password) >= 8 and
             any(c.islower() for c in password) and
             any(c.isupper() for c in password) and
             any((c.isalpha()==False) for c in password)) #Number or Special character
 
+
 PASSWORD_ERROR_MESSAGE =  {'security': ['Passwort muss mindestens acht Zeichen, einen Gross-, einen Kleinbuchstaben und entweder eine Zahl oder ein Sondernzeichen enthalten!']}       
 
+
 def odsh_user_create(context, data_dict):
     model = context['model']
     password = data_dict.get('password')
@@ -69,6 +112,7 @@ def odsh_user_create(context, data_dict):
     else:
         raise logic.ValidationError(PASSWORD_ERROR_MESSAGE)          
   
+
 def tpsh_user_update(context, data_dict):
       password = data_dict.get('password')
       if not password:
@@ -78,8 +122,6 @@ def tpsh_user_update(context, data_dict):
       return user_update(context, data_dict)
 
 
-
-
 @toolkit.side_effect_free
 def autocomplete(context, data_dict):
     query = {
@@ -102,3 +144,41 @@ def autocomplete(context, data_dict):
         filtered_suggestions.append(suggestion)
     final_suggestions = list(sorted(set(filtered_suggestions), key=filtered_suggestions.index))[:5]
     return final_suggestions
+
+
+def odsh_resource_create(context, data_dict):
+    is_linked_resource = not isinstance(data_dict['upload'], cgi.FieldStorage)
+    if is_linked_resource:
+        _download_linked_resource_to_tmp(data_dict['url'])
+        _emulate_file_upload(data_dict)
+    return resource_create(context, data_dict)
+
+TMP_FILE_PATH = '/tmp/temp_file_upload'
+
+def _download_linked_resource_to_tmp(url):
+    log.debug('Downloading linked resource from {}.'.format(url))
+    setup_proxy()
+    test_file = urllib2.urlopen(url).read()
+    clear_proxy()
+    with open(TMP_FILE_PATH, 'wb') as temporary_file:
+        temporary_file.write(test_file)
+
+def _emulate_file_upload(data_dict):
+    '''
+    This function updates the data_dict in order to emulate
+    the behaviour of resource creation with uploaded file for
+    resource creation with link
+    '''
+    temporary_file = open(TMP_FILE_PATH, 'rb')
+    upload_file = temporary_file
+    filename = data_dict['name']
+    upload = cgi.FieldStorage()
+    upload.disposition = 'form-data'
+    upload.disposition_options = {'filename': filename, 'name': 'upload'}
+    upload.fp = upload_file
+    upload.file = upload_file
+    upload.type = 'application/pdf'
+    upload.headers['content-type'] = upload.type
+    upload.name = 'upload'
+    upload.filename = filename
+    data_dict['upload'] = upload
\ No newline at end of file

ckanext/odsh/plugin.py

@@ -51,7 +51,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
     def get_actions(self):
         return {'package_create': action.odsh_package_create,
                 'user_update':action.tpsh_user_update,
-                'user_create': action.odsh_user_create}
+                'user_create': action.odsh_user_create,
+		'resource_create': action.odsh_resource_create,}
 
     
     # IConfigurer
@@ -127,7 +128,14 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 toolkit.get_validator('ignore_missing'),
                 toolkit.get_converter('convert_to_extras')
             ],
-
+            'relatedPackage': [
+                toolkit.get_validator('tpsh_validate_relatedPackage'),
+                toolkit.get_converter('convert_to_extras')
+            ],
+            'accessibility': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_converter('convert_to_extras')
+            ]
         })
         return schema
     
@@ -141,6 +149,14 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 toolkit.get_converter('convert_from_extras'),
                 toolkit.get_validator('ignore_missing')
             ],
+            'relatedPackage': [
+                toolkit.get_converter('convert_from_extras'),
+                toolkit.get_validator('ignore_missing')
+            ],
+            'accessibility': [
+                toolkit.get_converter('convert_from_extras'),
+                toolkit.get_validator('ignore_missing')
+            ],
         })
         return schema
 
@@ -320,7 +336,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         # Template helper function names should begin with the name of the
         # extension they belong to, to avoid clashing with functions from
         # other extensions.
-        return {'odsh_main_groups': odsh_helpers.odsh_main_groups,
+        return {
+	        'odsh_main_groups': odsh_helpers.odsh_main_groups,
                 'odsh_now': odsh_helpers.odsh_now,
                 'odsh_group_id_selected': odsh_helpers.odsh_group_id_selected,
                 'odsh_get_facet_items_dict': odsh_helpers.odsh_get_facet_items_dict,
@@ -357,6 +374,7 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 'tpsh_get_resource_size': helpers_tpsh.get_resource_size,
                 'tpsh_get_address_org':helpers_tpsh.get_address_org,
                 'tpsh_get_body_mail':helpers_tpsh.get_body_mail,
+		'tpsh_git_commit_hash': helpers_tpsh.git_commit_hash,
          }
 
     

ckanext/odsh/profiles/odsh_dcat_de_profile.py

@@ -10,7 +10,6 @@ from ckanext.dcatde.profiles import DCATdeProfile, DCATDE, DCAT, DCATDE_1_0
 
 import ckanext.odsh.helpers_tpsh as helpers_tpsh
 import ckanext.odsh.collection.helpers as helpers_collection
-from ckanext.odsh.helper_pkg_dict import HelperPgkDict
 
 
 DCT = rdflib.namespace.Namespace("http://purl.org/dc/terms/")
@@ -134,9 +133,7 @@ class ODSHDCATdeProfile(DCATdeProfile):
         return is_collection
     
     def _get_dataset_refs_belonging_to_collection(self, dataset_dict):
-        dataset_names = helpers_collection.get_all_datasets_belonging_to_collection(
-            collection_name = dataset_dict.get('id')
-        )
+        dataset_names = helpers_collection.get_dataset_names(dataset_dict)
 	dataset_dicts = [model.Package.get(name).as_dict() for name in dataset_names]
         dataset_ids = [dataset_dict.get('id') for dataset_dict in dataset_dicts]
         dataset_refs = [self._construct_refs(id) for id in dataset_ids]
@@ -155,12 +152,11 @@ class ODSHDCATdeProfile(DCATdeProfile):
         '''
         if dataset_dict.get('type')=='collection':
             return False
-        id_dataset = dataset_dict.get('id')
-        collection_name = helpers_collection.get_collection_name_by_dataset(id_dataset)
+        collection_name = helpers_collection.get_collection_id(dataset_dict)
 	return collection_name is not None
 
     def _add_collection(self, dataset_dict, dataset_ref):
-        collection_id = HelperPgkDict(dataset_dict).get_collection_id()
+        collection_id = helpers_collection.get_collection_id(dataset_dict)
         collection_uri = self._construct_refs(collection_id)
         self.g.set(
             (dataset_ref, DCT.isVersionOf, 

ckanext/odsh/public/odsh.css

@@ -130,6 +130,11 @@ h3{
     }
 }
 
+    .site-collection-summary .odsh-dataset-heading {
+        margin-left: 0px;
+        margin-right: 0px;
+    }
+
 .search-form, .odsh-dataset-heading {
     display: flex;
     align-items: center;
@@ -402,6 +407,11 @@ input[type=radio], input[type=checkbox] {
     max-width: 300px;
     padding: 40.66px 16px 0px;
 }
+
+.site-collection-summary .secondary.span3 {
+    display: none;
+}
+
 .secondary .module-narrow .module-content{
     padding-left: 0px;
     padding-right: 0px;
@@ -411,6 +421,10 @@ input[type=radio], input[type=checkbox] {
     padding-left: 31px;
 }
 
+.site-collection-summary .primary {
+    padding-left: 0;
+}
+
 .primary.organization {
     padding-left: 0px;
 }
@@ -1270,6 +1284,7 @@ body {
 .footer-icon{
     float: left;
     margin-right: 40px;
+    margin-bottom: 1rem;
 }
 
 .footer-icon.last{
@@ -1314,7 +1329,7 @@ body {
         box-sizing: border-box;
         padding-bottom: 15px;
         background-image: none;
-        height: 139px;
+        height: 150px;
         display: flex;
         flex-direction: column-reverse;
     }
@@ -1453,6 +1468,24 @@ body {
     color: #000000;
 }
 
++.hint-newer-version {
+    margin-top: 30px;
+    margin-bottom: 30px;
+    font-size: 20px;
+    color: white;
+    background-color: #D4004B;
+    padding: 0.5rem 1rem;
+}
+
+.hint-newer-version > a {
+    color: white;
+    text-decoration: underline;
+}
+
+.hint-newer-version > a:hover {
+    color: lightgray;
+}
+
 .resource-list {
     margin: 0px;
 }
@@ -2770,3 +2803,7 @@ body.filters-modal div.row > aside.secondary.span3 {
         margin-right: 6px;
     }
 }
+
+.tpsh-collection-list {
+    list-style-type: none;
+}

ckanext/odsh/public/odsh_header.css

@@ -242,7 +242,7 @@
 }
 
 @media (min-width: 768px) {
-    .navbar.masthead .navigation .nav-pills li:not(:first-child) {
+    .navbar.masthead .navigation .nav-pills li:not(:nth-child(2)) {
         margin-left: 22px;
     }
 }

ckanext/odsh/setup_proxy.py

+import urllib2
+from ckan.common import config
+
+
+def setup_proxy():
+    '''
+    This function declares that a proxy server shall be used to access the web via
+    urllib2. It takes the proxy address from ckan's config file 
+    (
+        field "ckanext.odsh.download_proxy",
+        example: ckanext.odsh.download_proxy = http://1.2.3.4:4123
+    )
+    '''
+
+    proxy_url = config.get('ckanext.odsh.download_proxy', None)
+    if proxy_url:
+        proxy = urllib2.ProxyHandler({'http': proxy_url, 'https': proxy_url})
+        opener = urllib2.build_opener(proxy)
+        urllib2.install_opener(opener)
+
+def clear_proxy():
+    proxy = urllib2.ProxyHandler({})
+    opener = urllib2.build_opener(proxy)
+    urllib2.install_opener(opener)
\ No newline at end of file

ckanext/odsh/templates/footer.html

@@ -12,6 +12,7 @@
             <div class="footer-right">
                 <div class='footer-icon'><a href='http://www.schleswig-holstein.de/tpkontakt'>Kontakt</a></div>
                 <div class='footer-icon'><a href='http://www.schleswig-holstein.de/tpimpressum'>Impressum</a></div>
+                <div class='footer-icon'><a href='https://www.schleswig-holstein.de/DE/Serviceseiten/Barrierefreiheitserklaerung/barrierefreiheit_node.html'>Barrierefreiheit</a></div>
                 <div class='footer-icon last'><a href='http://www.schleswig-holstein.de/tpdatenschutz'>Datenschutz</a></div>
             </div>
         </div>

ckanext/odsh/templates/package/collection_read.html

+{% extends "package/read.html" %}
+
+{% block bodytag %} {{ super() }} class="site-collection-summary" {% endblock %}
+
+{% block package_resources %}
+{% endblock package_resources %}
+
+{% block secondary_content %}
+{% endblock secondary_content %}
+
+{% block collection %}
+
+{% set collection_id = pkg.get('id') %}
+{% set collection = h.get_collection_info(collection_id) %}
+{% set collection_title = collection['title'] if collection else None %}
+
+{% if collection['members'] %}
+<p>
+    Dies ist die Übersichtsseite der Dokumentensammlung {{ collection_title }}. 
+    Sie enthält folgende Dokumente (neuere Dokumente zuerst):
+</p>
+{% snippet 'snippets/package_list.html', packages=collection['members'] | reverse() %}
+{% endif %}
+
+{% endblock collection %}

ckanext/odsh/templates/package/read.html

 {% extends "package/read_base.html" %}
 
 {% set pkg = c.pkg_dict %}
-{% set collection_title = h.collection_get_title(pkg) %}
-{% set successor_url = h.collection_get_successor(pkg) %}
-{% set predecessor_url = h.collection_get_predecessor(pkg) %}
-{% set latest_collection_member = h.collection_get_latest_member(pkg) %}
+{% set collection = h.get_collection(pkg) %}
+{% set collection_title = collection['title'] if collection else None %}
+{% set collection_url = collection['url'] if collection else None %}
+{% set successor_url = collection['successor']['url'] if collection else None %}
+{% set predecessor_url = collection['predecessor']['url'] if collection else None %}
+{% set latest_collection_member = collection['persistent_link_last_member'] if collection else None %}
 {% set stars = h.odsh_openness_score_dataset_html(pkg) %}
 
 {% block breadcrumb_content %}
@@ -22,7 +24,7 @@
 <li>{% link_for _('Datasets'), controller='package', action='search' %}</li>
 <li class="active"><a href="">{{ _('Create Dataset') }}</a></li>
 {% endif %}
-{% endblock %}
+{% endblock breadcrumb_content %}
 
 {% block primary_content_inner %}
 {{ super() }}
@@ -37,7 +39,7 @@
     {% if pkg.state == 'deleted' %}
     [{{ _('Deleted') }}]
     {% endif %}
-    {% endblock %}
+    {% endblock page_heading%%}
     <div class="odsh-dataset-edit-button">
       {% if h.check_access('package_update', {'id':pkg.id }) %}
       <div>
@@ -68,6 +70,12 @@
     <img src="/base/images/icon_info.svg" aria-hidden="true"></img>
     {{ _('Detailinformationen') }}
 </div>
+{% if successor_url %}
+<p class="hint-newer-version">
+    Hinweis: Es ist eine <a href="{{ latest_collection_member }}">
+    neuere Version</a> dieses Dokuments verfgbar.
+</p>
+{% endif %}
 {% if pkg.notes %}
 <div class="notes embedded-content">
   {{ h.render_markdown(h.get_translated(pkg, 'notes')) }}
@@ -83,14 +91,14 @@
         {% link_for _('Add new resource'), controller='package', action='new_resource', id=c.pkg_dict.name, class_='btn btn-primary', icon='plus' %}
       </div>
   {% endif %}
-{% endblock %}
+{% endblock package_resources %}
 
 {% block collection %}
 {% if latest_collection_member %}
 <section id="dataset-collection">
     {% if collection_title %}
     <p>
-        Dieser Datensatz ist Bestandteil der Datenreihe "{{ collection_title }}". 
+        Dieses Dokument ist Bestandteil der <a href={{ collection_url }}>Dokumentengruppe "{{ collection_title }}"</a>.
         Sie können zu älteren und neueren Datensätzen blättern.
     </p>
     {% endif %} {# collection_title #}
@@ -135,5 +143,5 @@
 {% endfor %}
 </div>
 
-{% endblock %}
+{% endblock primary_content_inner %}
 

ckanext/odsh/templates/snippets/package_item.html

@@ -22,10 +22,10 @@ Example:
 {% set license_name=' ('+licenseAttributionByText +')' if licenseAttributionByText else ''%}
 {% set license = license_title + license_name %}
 {% set org = package.organization.title or package.organization.name or '-' %}
-{% set stars = h.odsh_openness_score_dataset_html(package) %}
 {% set access_count=(package.tracking_summary.total if package.tracking_summary) or '-' %}
 {% set issued_extra = h.odsh_extract_value_from_extras(package.extras,'issued') %}
 {% set issued = h.odsh_render_datetime(issued_extra) if issued_extra else h.odsh_render_datetime(package.metadata_created)%}
+{% set subject_text = h.odsh_extract_value_from_extras(package.extras,'subject_text') if h.odsh_extract_value_from_extras(package.extras,'subject_text') else '-'  %}
 {% set daterange = h.tpsh_get_daterange_prettified(package) %}
 {% set language_of_package = h.tpsh_get_language_of_package(package) %}
 {% set language_icon = h.get_language_icon(package) %}
@@ -33,6 +33,18 @@ Example:
 
 {% block package_item %}
 <div class="odsh-dataset-item">
+    <div class="preview-image-container">
+            {% if thumbnail %}
+            <a href={{  h.url_for(controller='package', action='read', id=package.name) }}>
+                <img src= "/{{ thumbnail }}" alt= "Vorschau" />
+            </a>
+
+            {% else %}
+            <a href={{  h.url_for(controller='package', action='read', id=package.name) }}>
+                <img src="/base/images/platzhalter.svg" alt= "keine Vorschau verfügbar" />
+            </a>
+            {% endif%}
+    </div>
     {% block content %}
     <div class="dataset-content">
         {% block heading %}
@@ -107,40 +119,10 @@ Example:
             {% endfor %}
         </p>
         {% endif %}
+        <p class='package-info-subject'>{{ _('subject') }}: {{ subject_text }} </p>
+        <p class='package-info-issued'>{{ _('issued') }}: {{issued}} </p>
         {% endblock notes %}
     </div>
-
-    <div class='dataset-spacer'></div>
-
-  <div class="dataset-meta">
-    {% block resources %}
-    {% block resources_outer %}
-    {% if package.resources and not hide_resources %}
-    <ul class="dataset-resources">
-      {% block resources_inner %}
-      {% for resource in h.dict_list_reduce(package.resources, 'format') %}
-      <li>
-        <a href="{{ h.url_for(controller='package', action='read', id=package.name) }}" class="label dataformat-label"
-          data-format="{{ resource.lower() }}">{{
-          resource }}</a>
-      </li>
-      {% endfor %}
-      {% endblock %}
-    </ul>
-    {% endif %}
-    <div class="dataset-stars">
-    {% if stars>-1%}
-      {% snippet "qa/stars.html", stars=stars %}
-    {% endif %}
-    </div>
-
-    <div class="dataset-info issued">
-      <p>{{ _('issued') }}: {{issued}} </p>
-    </div>
-    {% endblock %}
-    {% endblock %}
-  </div>
-
     {% endblock content %}
 </div>
 {% endblock package_item %}

ckanext/odsh/templates/user/login.html

 {% extends "page.html" %}
 
+{% set commit_hash = h.tpsh_git_commit_hash() %}
+
 {% block subtitle %} {{ _('Login') }} {% endblock %}
 
+{% block skip %}
+<p hidden>
+    version={{ commit_hash }}
+</p>
+{{ super() }}
+{% endblock skip %}
+
 {% block breadcrumb_content %}
 <li class="active">{{ h.nav_link(_('Login'), controller='user', action='login') }}</li>
 {% endblock %}

ckanext/odsh/tools.py

 import os
-from ckanext.odsh.pdf_to_thumbnail.thumbnail import get_filepath_to_resource
+from ckanext.odsh.pdf_to_thumbnail.thumbnail import get_resource_path
 from ckanext.odsh.lib.uploader import calculate_hash
 import ckan.plugins.toolkit as toolkit
 import magic
@@ -12,7 +12,7 @@ def add_attributes_resources(context, resource):
     i = 0
     for item in resources:    
         if item.get('id') == resource.get('id'):
-            path = get_filepath_to_resource(resource)
+            path = get_resource_path(resource)
             if os.path.exists(path):
                 with open(path, 'rb') as file:                  
                     

ckanext/odsh/validation.py

@@ -6,14 +6,13 @@ import urllib2
 import json
 from itertools import count
 from dateutil.parser import parse
+from pylons import config
 
 import ckan.plugins.toolkit as toolkit
 import ckan.model as model
 from ckan.lib.navl.dictization_functions import Missing
 
-from pylons import config
-
-import pdb
+from ckanext.odsh.helpers_tpsh import get_package_dict
 
 _ = toolkit._
 
@@ -191,7 +190,6 @@ def known_spatial_uri(key, data, errors, context):
         poly = None
 
         # some harvesters might import a polygon directly...
-        # pdb.set_trace()
         poly = _extract_value(data, 'spatial')
 
         has_old_uri = False
@@ -243,10 +241,13 @@ def known_spatial_uri(key, data, errors, context):
 
 def _copy_spatial_uri_temp_to_extras(data):
     '''
-    copy the field spatial_uri_temp originating 
+    copy the field spatial_uri_temp or
+    spatial_url_temp originating 
     from the user interface to extras
     '''
     spatial_uri = data.get(('__extras',)).get('spatial_uri_temp')
+    if spatial_uri is None:
+        spatial_uri = data.get(('__extras',)).get('spatial_url_temp')
     is_spatial_uri_in_extras = _extract_value(data, 'spatial_uri') is not None
     if not is_spatial_uri_in_extras:
         next_index = next_extra_index(data)
@@ -346,6 +347,12 @@ def validate_subject(key, flattened_data, errors, context):
         raise toolkit.Invalid(_('Subject must not be empty.'))
     flattened_data = _convert_subjectID_to_subjectText(subject_id, flattened_data)
 
+def validate_relatedPackage(data):
+    if data:
+        try:
+            get_package_dict(data)
+        except logic.NotFound:
+            raise toolkit.Invalid("relatedPackage: package '{}' not found".format(data))
 
 def get_validators():
     return {
@@ -354,4 +361,5 @@ def get_validators():
         'odsh_validate_extras': validate_extras,
         'validate_licenseAttributionByText': validate_licenseAttributionByText,
         'tpsh_validate_subject': validate_subject,
+	'tpsh_validate_relatedPackage': validate_relatedPackage,
     }

uls.conf

-inputfile = /var/log/ckan/bulk.log
-inputfile = /var/log/ckan/std/gather_consumer.log
-inputfile = /var/log/ckan/std/fetch_consumer.log
-
-#outputfile = ./logs/out.uls
-
-iformat0 = %F %T,%c%c%c ERROR [%q] got exception ...%W
-nreadformat0 =... : %W
-
-# 'got exception...' ist die erste Zeile eines Tracebacks
-reg0 = got exception ...  
-# alle forgenden Zeilen haben diesen Präfix
-nreadreg0 =^... :
-
-write0 = V;%F %T;%h;"odsh";"Exception";"traceback";"in %p
-nreadwrite0 = %W
\ No newline at end of file