Merge pull request #66 in TPSH/ckanext-odsh from refactor-collections to dev

* commit '907e38ac':
  fixes bug in plugin collections
  renames protected name format to resource_format
  fixes bugs in controller
  returns resource instead of name
  removes  profiler
  refactors latest_resource
  refactors get_latest_dataset
  removes deprecated helpers from collection plugin
  refactoring: extracts method get_collection_info from get_collection preparation for use in controller.py
  WIP: rewrite of collections plugin

ckanext/odsh/collection/controller.py

 from ckan.lib.helpers import is_url, url_for
 import ckan.plugins.toolkit as toolkit
 from ckan.controllers.package import PackageController
-from helpers import get_latest_resources_for_type, get_latest_dataset
+from helpers import get_latest_resources_for_format, get_latest_dataset
 
 
 
@@ -13,10 +13,10 @@ class LatestDatasetController(PackageController):
 
 class LatestRecourcesController(PackageController):
     
-    def latest_resource(self, id, type):
-        latest_resources = get_latest_resources_for_type(id, type)
+    def latest_resource(self, id, resource_format):
+        latest_resources = get_latest_resources_for_format(id, resource_format)
         if latest_resources is None:
-            abort(404)
+            toolkit.abort(404)
         url_type = latest_resources.get('url_type')
         if url_type is None:
             resource_url = latest_resources.get('url')
@@ -35,4 +35,4 @@ class LatestRecourcesController(PackageController):
                                     filename=pre_resource_url,
                                     qualified = True)
             toolkit.redirect_to(url_resource)
-        abort(404)
+        toolkit.abort(404)
\ No newline at end of file

ckanext/odsh/collection/helpers.py

 from string import lower
+from operator import itemgetter
 
 import ckan.lib.helpers as helpers
 import ckan.model as model
 import ckan.plugins.toolkit as toolkit
 
 
+def get_collection(dataset_dict):
+    collection_id = get_collection_id(dataset_dict)
+    if collection_id:
+        return get_collection_info(collection_id, dataset_dict)
+    
+    return None
+
+
+def get_collection_info(collection_id, dataset_dict=None):
+    collection_dict = get_package_dict(collection_id)
+    dataset_names = get_dataset_names(collection_dict)
+    datasets_in_collection = get_datasets_from_solr(dataset_names)
+    collection_info = gather_collection_info(collection_dict, datasets_in_collection, dataset_dict)
+    return collection_info
+
+
+def get_collection_id(dataset_dict):
+    relationships_dataset = dataset_dict.get('relationships_as_subject')
+    if len(relationships_dataset):        
+        return relationships_dataset[0]['__extras']['object_package_id']
+    return None
 
-#routine functions
 
 def get_package_dict(name):
     return model.Package.get(name).as_dict()
 
-def get_relationships(name):
-    collection_dict = get_package_dict(name)
-    return collection_dict.get('relationships')
 
-def get_all_datasets_belonging_to_collection(collection_name):
-    list_rel_collection = get_relationships(collection_name)
-    name_list = list()
-    for item in list_rel_collection:
-        item_object = item.get('object') 
-        name_list.append(item_object)
-    return name_list
+def get_dataset_names(collection_dict):
+    relationships_collection = collection_dict.get('relationships')
+    names_collection_members = [relationship.get('object') for relationship in relationships_collection]
+    return names_collection_members
 
-#for mapping latest resources and latest dataset
 
-def get_latest_dataset(collection_name):
-    collection_list_relationships = get_all_datasets_belonging_to_collection(collection_name)
-    latest_issued = latest_name = None
-    for item in collection_list_relationships:
-        item_pkt_dict =  get_package_dict(item)
-        if helpers.check_access('package_show', item_pkt_dict):
-            item_issued = item_pkt_dict.get('extras').get('issued')
-            if latest_issued < item_issued or (latest_issued == item_issued and latest_name < item):
-                latest_name=item
-                latest_issued=item_issued
-    return latest_name
+def get_datasets_from_solr(dataset_names):
+    context = None
 
+    name_expression = ' OR '.join(dataset_names)
+    fq = 'name:({})'.format(name_expression)
     
-def is_latest_resources(resource_format, type, resource_created,latest_created, resource_id, latest_id):
-    if lower(resource_format) == lower(type):
-        return (resource_created > latest_created or (resource_created == latest_created and resource_id > latest_id))
-    else:
-        return False
-
-def get_latest_resources_for_type(collection_name, type):
-    latest_dataset_name = get_latest_dataset(collection_name)
-    latest_dataset = get_package_dict(latest_dataset_name)
-    resource_list = latest_dataset.get('resources')
-    latest_resource = latest_created = latest_id = None
-    for resource in resource_list:
-        resource_format = resource.get('format')
-        resource_created = resource.get('created')
-        resource_id = resource.get('id')
-        if is_latest_resources(resource_format, type, resource_created, latest_created, resource_id, latest_id):
-            latest_id=resource_id
-            latest_created=resource_created
-            latest_resource=resource
-    return latest_resource
-
-#for predecessor and successor
-
-
-
-def get_collection_name_by_dataset(dataset_name):
-    list_rel_dataset = get_relationships(dataset_name)
-    if len(list_rel_dataset):        
-        return list_rel_dataset[0]['object']
-
-def get_collection_title_by_dataset(pkg_dict_dataset):
-    dataset_name = pkg_dict_dataset.get('name')
-    collection_name = get_collection_name_by_dataset(dataset_name)
-    if not collection_name:
-        return None
-    context = None
-    pkg_dict_collection = toolkit.get_action('package_show')(context, {'id': collection_name})
-    if not pkg_dict_collection:
+    sort = 'name asc,extras_issued asc'
+    
+    # maximum possible number of results is 1000, 
+    # see https://docs.ckan.org/en/ckan-2.7.3/api/index.html#ckan.logic.action.get.package_search
+    query_result = toolkit.get_action('package_search')(context, {
+        'fq': fq,
+        'sort': sort,
+        'rows': 1000, 
+    })
+
+    results = query_result.get('results')
+    datasets_found = results if results else []
+
+    return datasets_found
+
+
+def gather_collection_info(collection_dict, datasets_in_collection, dataset_dict=None):
+    url_from_id = lambda id: helpers.url_for(controller='package', action='read', id=id)
+
+
+    name_first_dataset = datasets_in_collection[0].get('name')
+    url_first_dataset = url_from_id(name_first_dataset)
+    
+    name_last_dataset = datasets_in_collection[-1].get('name')
+    url_last_dataset = url_from_id(name_last_dataset)
+
+    name_collection = collection_dict.get('name')
+    persistent_link_last_member = helpers.url_for(
+        controller='ckanext.odsh.collection.controller:LatestDatasetController', 
+        action='latest',
+        id=name_collection
+    )
+
+    if dataset_dict:
+        name_current_dataset = dataset_dict.get('name')
+        dataset_names = [d.get('name') for d in datasets_in_collection]
+        
+        def get_predecessor():
+            id_current = dataset_names.index(name_current_dataset)
+            if id_current and id_current > 0:
+                return dataset_names[id_current - 1]
             return None
-    title_collection = pkg_dict_collection.get('title')
-    return title_collection
-
-def get_all_datasets_belonging_to_collection_by_dataset(dataset_name):
-    collection_name = get_collection_name_by_dataset(dataset_name)
-    if collection_name: 
-        return get_all_datasets_belonging_to_collection(collection_name)
-    return []
-
-def _get_siblings_dicts_with_access(pkg_dict):
-    dataset_name = pkg_dict.get('name')
-    list_of_siblings = get_all_datasets_belonging_to_collection_by_dataset(dataset_name)
-    n_siblings = len(list_of_siblings)
-    if n_siblings>0:
-        siblings_dicts = [get_package_dict(name) for name in list_of_siblings]
-        user_has_access = lambda pkg_dict:helpers.check_access('package_show', pkg_dict)
-        siblings_dicts_with_access = filter(user_has_access, siblings_dicts)
-        return siblings_dicts_with_access
+        
+        def get_successor():
+            id_current = dataset_names.index(name_current_dataset)
+            if id_current and id_current < len(dataset_names) - 1:
+                return dataset_names[id_current + 1]
             return None
         
-def _sort_siblings_by_name_and_date(siblings_dicts):
-    '''
-    sort by name first and then by date to have a fallback if dates are the same
-    '''
-    _get_name = lambda pkg_dict:pkg_dict.get('name')
-    _get_issued = lambda pkg_dict:pkg_dict.get('extras').get('issued')
-    siblings_dicts_sorted_by_name = sorted(siblings_dicts, key=_get_name)
-    siblings_dicts_sorted_by_date_issued = sorted(siblings_dicts_sorted_by_name, key=_get_issued)
-    return siblings_dicts_sorted_by_date_issued
-
-def get_successor_and_predecessor_dataset(pkg_dict):
-    dataset_name = pkg_dict.get('name')
-    siblings_dicts_with_access = _get_siblings_dicts_with_access(pkg_dict)
-    if siblings_dicts_with_access:
-        n_siblings = len(siblings_dicts_with_access)
-        siblings_dicts_sorted_by_date_issued = _sort_siblings_by_name_and_date(siblings_dicts_with_access)
-        siblings_names_sorted_by_date_issued = [d['name'] for d in siblings_dicts_sorted_by_date_issued]
-        id_current_dataset = siblings_names_sorted_by_date_issued.index(dataset_name)
-        predecessor_name = (
-            siblings_names_sorted_by_date_issued[id_current_dataset-1] if (id_current_dataset > 0) 
-            else None
-        )
-        successor_name = (
-            siblings_names_sorted_by_date_issued[id_current_dataset+1] if (id_current_dataset < n_siblings-1) 
-            else None
-        )
+        name_predecessor = get_predecessor()
+        url_predecessor = url_from_id(name_predecessor) if name_predecessor else None
+        
+        name_successor = get_successor()
+        url_successor = url_from_id(name_successor) if name_successor else None
     else:
-        predecessor_name, successor_name = None, None
-    return successor_name, predecessor_name
-
-def get_successor_and_predecessor_urls(pkg_dict):
-    successor_name, predecessor_name = get_successor_and_predecessor_dataset(pkg_dict)
-    successor_url, predecessor_url = (
-        helpers.url_for(controller='package', action='read', id=name)
-        if name is not None
-        else None
-        for name in (successor_name, predecessor_name)
-    )
-    return successor_url, predecessor_url
+        url_predecessor = url_successor = None
+    
+    return {
+        'title': collection_dict.get('title'),
+        'members': datasets_in_collection,
+        'first_member': {
+            'name': name_first_dataset,
+            'url': url_first_dataset,
+        },
+        'last_member': {
+            'name': name_last_dataset,
+            'url': url_last_dataset,
+        },
+        'predecessor': {
+            'url': url_predecessor,
+        },
+        'successor': {
+            'url': url_successor,
+        },
+        'persistent_link_last_member': persistent_link_last_member,
+    }
 
-def get_successor(pkg_dict):
-    successor_and_predecessor = get_successor_and_predecessor_urls(pkg_dict)
-    return successor_and_predecessor[0]
 
-def get_predecessor(pkg_dict):
-    successor_and_predecessor = get_successor_and_predecessor_urls(pkg_dict)
-    return successor_and_predecessor[1]
+def get_latest_dataset(collection_name):
+    collection_info = get_collection_info(collection_name)
+    latest_name = collection_info['last_member']['name']
+    return latest_name
 
-#link to latest collection member
-def latest_collection_member_persistent_link(pkg_dict):
-    dataset_name = pkg_dict.get('name')
-    collection_name = get_collection_name_by_dataset(
-        dataset_name=dataset_name
-    )
-    if not collection_name:
+
+def get_latest_resources_for_format(collection_name, resource_format):
+    collection_info = get_collection_info(collection_name)
+    members = collection_info.get('members')
+    if not members:
         return None
-    url = helpers.url_for(
-        controller='ckanext.odsh.collection.controller:LatestDatasetController', 
-        action='latest',
-        id=collection_name
-    )
-    return url
+    latest_dataset = members[-1]
+    resources = latest_dataset.get('resources')
+    if not resources:
+        return None
+    resources_with_asked_type = [r for r in resources if r.get('format').upper() == resource_format.upper()]
+    resources_sorted = sorted(resources_with_asked_type, key=itemgetter('id','created'), reverse=True)
\ No newline at end of file
+    return resources_sorted[-1]

ckanext/odsh/collection/plugin.py

@@ -28,7 +28,7 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
         )
 
         map.connect(
-            '/collection/{id}/aktuell.{type}',
+            '/collection/{id}/aktuell.{resource_format}',
             controller='ckanext.odsh.collection.controller:LatestRecourcesController',
             action='latest_resource'
         )
@@ -39,7 +39,7 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
             path_prefix='/collection/'
         ) as m:
             m.connect('latest', '{id}/aktuell', action='latest')
-            m.connect('latest_resource', '{id}/aktuell.{type}', action='latest_resource')  
+            m.connect('latest_resource', '{id}/aktuell.{resource_format}', action='latest_resource')  
         return map
 
     def after_map(self, map):
@@ -53,8 +53,5 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
         # other extensions.
 
         return {
-            'collection_get_successor': collection_helpers.get_successor,
-            'collection_get_predecessor': collection_helpers.get_predecessor,
-            'collection_get_latest_member':collection_helpers.latest_collection_member_persistent_link,
-            'collection_get_title': collection_helpers.get_collection_title_by_dataset,
+            'get_collection': collection_helpers.get_collection,
         }

ckanext/odsh/helper_pkg_dict.py

@@ -108,7 +108,7 @@ class HelperPgkDict(object):
         the containing collection
         '''
         package_name = self.pkg_dict.get('name')
-        collection_name = helpers_collection.get_collection_name_by_dataset(package_name)
+        collection_name = helpers_collection.get_collection_id(package_name)
         collection_dict = helpers_collection.get_package_dict(collection_name)
         collection_id = collection_dict.get('id')
         return collection_id

ckanext/odsh/profiles/odsh_dcat_de_profile.py

@@ -176,7 +176,7 @@ class ODSHDCATdeProfile(DCATdeProfile):
         if dataset_dict.get('type')=='collection':
             return False
         id_dataset = dataset_dict.get('id')
-        collection_name = helpers_collection.get_collection_name_by_dataset(id_dataset)
+        collection_name = helpers_collection.get_collection_name(id_dataset)
         return collection_name is not None
 
     def _add_collection(self, dataset_dict, dataset_ref):

ckanext/odsh/templates/package/read.html

 {% extends "package/read_base.html" %}
 
 {% set pkg = c.pkg_dict %}
-{% set collection_title = h.collection_get_title(pkg) %}
-{% set successor_url = h.collection_get_successor(pkg) %}
-{% set predecessor_url = h.collection_get_predecessor(pkg) %}
-{% set latest_collection_member = h.collection_get_latest_member(pkg) %}
+{% set collection = h.get_collection(pkg) %}
+{% set collection_title = collection['title'] if collection else None %}
+{% set successor_url = collection['successor']['url'] if collection else None %}
+{% set predecessor_url = collection['predecessor']['url'] if collection else None %}
+{% set latest_collection_member = collection['persistent_link_last_member'] if collection else None %}
 {% set thumbnail_url = h.thumbail_get_download_link(pkg) %}
 
 {% block breadcrumb_content %}