Skip to content
Snippets Groups Projects
Commit ed80c273 authored by Thorge Petersen's avatar Thorge Petersen
Browse files

Added search_facets to facet list call and fixed organization facets

parent 28d10833
No related branches found
No related tags found
1 merge request!50Upgraded to CKAN 2.10
Show whitespace changes
Inline Side-by-side
ckanext/odsh/helpers.py
+ 145
− 81
View file @ ed80c273
......@@ -8,20 +8,23 @@ import ckan.logic as logic
import ckan.model as model
import ckan.lib.helpers as helpers
import json
from ckan.common import c
import datetime
from dateutil import parser
from ckan.common import config
import urllib.request, urllib.parse, urllib.error
import urllib.request
import urllib.parse
import urllib.error
import hashlib
import re
import csv
import urllib.request, urllib.error, urllib.parse
from ckan.common import request
import urllib.request
import urllib.error
import urllib.parse
from ckan.common import config, request, g, c
from urllib.parse import urlsplit, urlunsplit
import ckan.lib.helpers as helpers
from collections import OrderedDict
import pkg_resources
from typing import Any, Optional
CKAN_TYPES = {'http://dcat-ap.de/def/datasetTypes/collection': 'collection'}
......@@ -121,6 +124,7 @@ def odsh_get_spatial_text(pkg_dict):
return spatial
return None
def extend_search_convert_local_to_utc_timestamp(str_timestamp):
if not str_timestamp:
return None
......@@ -132,6 +136,7 @@ def extend_search_convert_local_to_utc_timestamp(str_timestamp):
return dt+"Z"
def odsh_render_datetime(datetime_, fromIso=True):
date_format = '{0.day:02d}.{0.month:02d}.{0.year:04d}'
if not datetime_:
......@@ -209,6 +214,7 @@ def odsh_extract_error(key, errors, field='extras'):
if error.startswith(key):
return error
def odsh_extract_error_new(key, errors):
if not errors or not ('__extras' in errors):
return None
......@@ -216,6 +222,7 @@ def odsh_extract_error_new(key, errors):
if error:
return key + ': ' + error
def odsh_extract_value_from_extras(extras, key):
if not extras:
return None
......@@ -241,17 +248,20 @@ def presorted_license_options(existing_license_id=None):
for license_id in license_ids]
def odsh_has_more_facets(facet, limit=None, exclude_active=False):
def odsh_has_more_facets(facet: str,
search_facets: dict[str, dict[str, Any]],
limit: Optional[int] = None,
exclude_active: bool = False) -> bool:
facets = []
for facet_item in c.search_facets.get(facet)['items']:
for facet_item in search_facets.get(facet)['items']:
if not len(facet_item['name'].strip()) or facet_item['count'] == 0:
continue
if not (facet, facet_item['name']) in list(request.args.items()):
facets.append(dict(active=False, **facet_item))
elif not exclude_active:
facets.append(dict(active=True, **facet_item))
if c.search_facets_limits and limit is None:
limit = c.search_facets_limits.get(facet)
if getattr(g, 'search_facets_limits', None) and limit is None:
limit = g.search_facets_limits.get(facet)
if limit is not None and len(facets) > limit:
return True
return False
......@@ -260,9 +270,11 @@ def odsh_has_more_facets(facet, limit=None, exclude_active=False):
def odsh_public_url():
return config.get('ckanext.odsh.public_url')
def spatial_extends_available():
extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
file_path = config.get('ckanext.odsh.spatial.mapping', extension_path + '/resources/schleswig-holstein_geojson.csv')
file_path = config.get('ckanext.odsh.spatial.mapping',
extension_path + '/resources/schleswig-holstein_geojson.csv')
try:
mapping_file = urllib.request.urlopen(file_path)
......@@ -278,6 +290,7 @@ def spatial_extends_available():
result.append(spatial_text.decode('UTF-8'))
return result
def odsh_public_resource_url(res):
home = config.get('ckanext.odsh.public_url')
if res.get('url_type', None) == 'upload' and 'url' in res:
......@@ -355,7 +368,8 @@ def is_within_last_month(date, date_ref=None):
if not date_ref:
date_ref = datetime.date.today()
[year_ref, month_ref, day_ref] = [date_ref.year, date_ref.month, date_ref.day]
[year_ref, month_ref, day_ref] = [
date_ref.year, date_ref.month, date_ref.day]
try:
if month_ref > 1:
......@@ -364,17 +378,20 @@ def is_within_last_month(date, date_ref=None):
one_month_ago = datetime.date(year_ref-1, 12, day_ref)
except ValueError:
# this happens if month before month_ref has less days than month_ref
one_month_ago = datetime.date(year_ref, month_ref, 1) - datetime.timedelta(days=1)
one_month_ago = datetime.date(
year_ref, month_ref, 1) - datetime.timedelta(days=1)
if date > one_month_ago:
return True
return False
def get_all_datasets_belonging_to_collection(context, collection_name):
rel_collection_dict = dict({"id": collection_name})
name_list = list()
try:
list_rel_collection = get_action('package_relationships_list')(context, rel_collection_dict)
list_rel_collection = get_action(
'package_relationships_list')(context, rel_collection_dict)
except AssertionError:
# if there does not exist an relationship, returns an empty list
return name_list
......@@ -383,29 +400,38 @@ def get_all_datasets_belonging_to_collection(context, collection_name):
name_list.append(item_object)
return name_list
def get_all_datasets_belonging_to_collection_by_dataset(context, dataset_name):
collection_name = get_collection_name_by_dataset(context, dataset_name)
if collection_name:
name_list = get_all_datasets_belonging_to_collection(context, collection_name)
name_list = get_all_datasets_belonging_to_collection(
context, collection_name)
return name_list
return list()
def get_collection_name_by_dataset(context, dataset_name):
rel_dataset_dict = dict({"id": dataset_name})
list_rel_dataset = toolkit.get_action('package_relationships_list')(context, rel_dataset_dict)
list_rel_dataset = toolkit.get_action(
'package_relationships_list')(context, rel_dataset_dict)
if not len(list_rel_dataset):
return None
collection_name = list_rel_dataset[0]['object']
return collection_name
def get_successor_and_predecessor_dataset(context, pkg_dict):
dataset_name = pkg_dict.get('name')
siblings_dicts_with_access = _get_siblings_dicts_with_access(context, pkg_dict)
siblings_dicts_with_access = _get_siblings_dicts_with_access(
context, pkg_dict)
if siblings_dicts_with_access:
n_siblings = len(siblings_dicts_with_access)
siblings_dicts_sorted_by_date_issued = _sort_siblings_by_name_and_date(siblings_dicts_with_access)
siblings_names_sorted_by_date_issued = [d['name'] for d in siblings_dicts_sorted_by_date_issued]
id_current_dataset = siblings_names_sorted_by_date_issued.index(dataset_name)
siblings_dicts_sorted_by_date_issued = _sort_siblings_by_name_and_date(
siblings_dicts_with_access)
siblings_names_sorted_by_date_issued = [
d['name'] for d in siblings_dicts_sorted_by_date_issued]
id_current_dataset = siblings_names_sorted_by_date_issued.index(
dataset_name)
predecessor_name = (
siblings_names_sorted_by_date_issued[id_current_dataset-1] if (id_current_dataset > 0)
else None
......@@ -418,14 +444,18 @@ def get_successor_and_predecessor_dataset(context, pkg_dict):
predecessor_name, successor_name = None, None
return successor_name, predecessor_name
def _get_siblings_dicts_with_access(context, pkg_dict):
dataset_name = pkg_dict.get('name')
list_of_siblings = get_all_datasets_belonging_to_collection_by_dataset(context, dataset_name)
list_of_siblings = get_all_datasets_belonging_to_collection_by_dataset(
context, dataset_name)
n_siblings = len(list_of_siblings)
if n_siblings > 0:
siblings_dicts = [get_package_dict(name) for name in list_of_siblings]
user_has_access = lambda pkg_dict:helpers.check_access('package_show', pkg_dict)
siblings_dicts_with_access = list(filter(user_has_access, siblings_dicts))
def user_has_access(pkg_dict): return helpers.check_access(
'package_show', pkg_dict)
siblings_dicts_with_access = list(
filter(user_has_access, siblings_dicts))
return siblings_dicts_with_access
return None
......@@ -434,18 +464,22 @@ def _sort_siblings_by_name_and_date(siblings_dicts):
'''
sort by name first and then by date to have a fallback if dates are the same
'''
_get_name = lambda pkg_dict:pkg_dict.get('name')
_get_issued = lambda pkg_dict:odsh_extract_value_from_extras(pkg_dict.get('extras'), 'issued')
def _get_name(pkg_dict): return pkg_dict.get('name')
def _get_issued(pkg_dict): return odsh_extract_value_from_extras(
pkg_dict.get('extras'), 'issued')
siblings_dicts_sorted_by_name = sorted(siblings_dicts, key=_get_name)
siblings_dicts_sorted_by_date_issued = sorted(siblings_dicts_sorted_by_name, key=_get_issued)
siblings_dicts_sorted_by_date_issued = sorted(
siblings_dicts_sorted_by_name, key=_get_issued)
return siblings_dicts_sorted_by_date_issued
def get_package_dict(name):
return model.Package.get(name).as_dict()
def get_successor_and_predecessor_urls(context, pkg_dict):
successor_name, predecessor_name = get_successor_and_predecessor_dataset(context, pkg_dict)
successor_name, predecessor_name = get_successor_and_predecessor_dataset(
context, pkg_dict)
successor_url, predecessor_url = (
helpers.url_for('dataset.read', id=name)
if name is not None
......@@ -454,6 +488,7 @@ def get_successor_and_predecessor_urls(context, pkg_dict):
)
return successor_url, predecessor_url
def short_name_for_category(category_name):
translations = {
'soci': 'Bevölkerung',
......@@ -472,6 +507,7 @@ def short_name_for_category(category_name):
}
return translations.get(category_name)
def odsh_load_mdk_sample_dataset():
'''
Load sample dataset (Musterkatalog/Musterdatensatz).
......@@ -487,9 +523,11 @@ def odsh_load_mdk_sample_dataset():
try:
with open(sample_data_file_path) as mapping_json:
MDK_MAPPING = json.loads(mapping_json.read(), object_pairs_hook=OrderedDict)
MDK_MAPPING = json.loads(
mapping_json.read(), object_pairs_hook=OrderedDict)
default = [{'value': 'Musterdatensatz wählen..', 'key': ''}]
mdk = [{'key': key, 'value': MDK_MAPPING[key]} for key in MDK_MAPPING]
mdk = [{'key': key, 'value': MDK_MAPPING[key]}
for key in MDK_MAPPING]
result = default+mdk
except IOError as err:
log.error(
......@@ -505,6 +543,7 @@ def odsh_load_mdk_sample_dataset():
raise
return result
def odsh_load_raw_mdk_sample_dataset():
'''
Load sample dataset (Musterkatalog/Musterdatensatz).
......@@ -520,7 +559,8 @@ def odsh_load_raw_mdk_sample_dataset():
try:
with open(sample_data_file_path) as mapping_json:
result = json.loads(mapping_json.read(), object_pairs_hook=OrderedDict)
result = json.loads(mapping_json.read(),
object_pairs_hook=OrderedDict)
except IOError as err:
log.error(
'Could not load sample dataset mapping file from {}'
......@@ -544,15 +584,18 @@ def map_dct_type_to_ckan_type(dct_type):
ckan_type = CKAN_TYPES.get(dct_type)
return ckan_type
def map_ckan_type_to_dct_type(ckan_type):
DCT_TYPES = _revert_dict(CKAN_TYPES)
dct_type = DCT_TYPES.get(ckan_type)
return dct_type
def _revert_dict(d):
d_inverse = {v: k for k, v in d.items()}
return d_inverse
def add_pkg_to_collection(id_pkg, id_collection):
if id_pkg and id_collection:
relationship_dict = {
......@@ -560,7 +603,9 @@ def add_pkg_to_collection(id_pkg, id_collection):
'object': id_collection,
'type': 'child_of',
}
toolkit.get_action('package_relationship_create')(None, relationship_dict)
toolkit.get_action('package_relationship_create')(
None, relationship_dict)
def correct_missing_relationship(pkg_dict, pkg_relationships_from_model):
'''
......@@ -585,23 +630,29 @@ def correct_missing_relationship(pkg_dict, pkg_relationships_from_model):
'id': relationship_from_model.id,
'type': relationship_from_model.type,
}
pkg_dict['relationships_as_subject'].append(relationship_for_package)
pkg_dict['relationships_as_subject'].append(
relationship_for_package)
return pkg_dict
def get_pkg_relationships_from_model(pkg_dict):
pkg_id = pkg_dict.get('id')
return model.Package.get(pkg_id).get_relationships()
def load_language_mapping():
extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
file_path = config.get('ckanext.odsh.language_mapping', extension_path + '/resources/language_mapping.json')
file_path = config.get('ckanext.odsh.language_mapping',
extension_path + '/resources/language_mapping.json')
with open(file_path) as language_mapping_json:
LANGUAGE_MAPPING = json.loads(language_mapping_json.read())
return LANGUAGE_MAPPING
def load_json_to_ordered_dict(json_str):
return json.loads(json_str, object_pairs_hook=OrderedDict)
def get_language_of_package(pkg_dict):
LANGUAGE_MAPPING = load_language_mapping()
language_id = _get_language_id(pkg_dict)
......@@ -610,6 +661,7 @@ def get_language_of_package(pkg_dict):
language = LANGUAGE_MAPPING.get(language_id)
return language
def get_language_icon(pkg_dict):
ICONS = {
"http://publications.europa.eu/resource/authority/language/DAN": '/base/images/icon_lang_danish.png',
......@@ -620,8 +672,10 @@ def get_language_icon(pkg_dict):
return None
return ICONS.get(language_id)
def _get_language_id(pkg_dict):
language_id = odsh_extract_value_from_extras(pkg_dict.get('extras'), 'language')
language_id = odsh_extract_value_from_extras(
pkg_dict.get('extras'), 'language')
language_id = pkg_dict.get('language')
if not language_id:
language_id = odsh_extract_value_from_extras(
......@@ -632,9 +686,11 @@ def _get_language_id(pkg_dict):
language_id_cleaned = re.sub('[\[\]\"]', '', language_id)
return language_id_cleaned
def get_spatial_for_selection():
extension_path = pkg_resources.resource_filename('ckanext.odsh', '')
file_path = config.get('ckanext.odsh.spatial.mapping', extension_path + '/resources/schleswig-holstein_geojson.csv')
file_path = config.get('ckanext.odsh.spatial.mapping',
extension_path + '/resources/schleswig-holstein_geojson.csv')
with open(file_path, newline='') as mapping_file:
cr = csv.reader(mapping_file, delimiter="\t")
spatial_mapping = list(cr)
......@@ -650,13 +706,15 @@ def get_spatial_for_selection():
else:
# Check if there is a municipality key entry for the value
municipality_entry = next(
(entry for entry in spatial_mapping if entry[1] == value and "municipalityKey" in entry[0]),
(entry for entry in spatial_mapping if entry[1]
== value and "municipalityKey" in entry[0]),
None
)
if municipality_entry:
# If a municipality key entry exists, use it instead of the current key
unique_mapping.append({'key': municipality_entry[0], 'value': value})
unique_mapping.append(
{'key': municipality_entry[0], 'value': value})
else:
# Otherwise, use the current key
unique_mapping.append({'key': key, 'value': value})
......@@ -669,9 +727,11 @@ def get_spatial_for_selection():
def get_language_for_selection():
LANGUAGE_MAPPING = load_language_mapping()
dict_for_select_box = [{'key': key, 'value': LANGUAGE_MAPPING[key]} for key in LANGUAGE_MAPPING]
dict_for_select_box = [
{'key': key, 'value': LANGUAGE_MAPPING[key]} for key in LANGUAGE_MAPPING]
return dict_for_select_box
def get_package_dict(name):
'''
raises ckan.logic.NotFound if not found
......@@ -679,6 +739,7 @@ def get_package_dict(name):
package_dict = toolkit.get_action('package_show')(None, {'id': name})
return package_dict
def size_of_fmt(num, suffix='B'):
for unit in ['', ' k', ' M', ' G', ' T', ' P', ' E', ' Z']:
if abs(num) < 1000.0:
......@@ -686,6 +747,7 @@ def size_of_fmt(num, suffix='B'):
num /= 1000.0
return "%.1f%s%s" % (num, 'Y', suffix)
def get_resource_size(resource):
resource_size = resource.get('size')
if resource_size:
......@@ -710,9 +772,11 @@ def get_body_mail(organization, package):
package_name = package.get('name')
url = helpers.url_for('dataset.read', id=package_name, qualified=True)
title = package.get('title')
anrede = "Sehr geehrte Damen und Herren," + "%0D%0A" + "%0D%0A" + "zu folgendem Eintrag habe ich eine Anmerkung/Frage:" + "%0D%0A" + "%0D%0A"
anrede = "Sehr geehrte Damen und Herren," + "%0D%0A" + "%0D%0A" + \
"zu folgendem Eintrag habe ich eine Anmerkung/Frage:" + "%0D%0A" + "%0D%0A"
mail_titel = "Titel: " + title + "%0D%0A"
mail_document = "Dokument-ID: " + package_name + "%0D%0A"
mail_url = "URL: " + url + "%0D%0A" + "%0D%0A"
message = mail_titel + mail_document + mail_url + "Mein Kommentar:" + "%0D%0A" + "%0D%0A" + "%0D%0A" + "%0D%0A"
message = mail_titel + mail_document + mail_url + \
"Mein Kommentar:" + "%0D%0A" + "%0D%0A" + "%0D%0A" + "%0D%0A"
return anrede + message
ckanext/odsh/templates/organization/index.html
+ 2
− 2
View file @ ed80c273
......@@ -24,7 +24,7 @@
{% block page_primary_action %}
{% endblock %}
{% block primary %}
{% block primary_content_inner %}
<div class="primary organization">
{% if h.check_access('organization_create') %}
{% link_for _('Add Organization'), named_route='organization.new', class_='btn btn-primary',
......@@ -56,7 +56,7 @@
</div>
{% endblock %}
{% block secondary%}
{% block secondary_content %}
{% endblock %}
{% block pre_wrap %}
......
ckanext/odsh/templates/organization/read.html
+ 39
− 39
View file @ ed80c273
{% extends "organization/read_base.html" %}
{% set dataset_type = h.default_package_type() %}
{% block page_primary_action %}
{% if h.check_access('package_create', {'owner_org': c.group_dict.id}) %}
{% link_for _('Add dataset'), named_route='dataset.new', group=c.group_dict.id, class_='btn btn-primary',
{% if h.check_access('package_create', {'owner_org': group_dict.id}) %}
{% link_for _('Add dataset'), named_route='dataset.new', group=group_dict.id, class_='btn btn-primary',
icon='plus-square' %}
{% endif %}
{% endblock %}
......@@ -33,26 +34,25 @@ icon='plus-square' %}
{% snippet 'snippets/search_form.html', form_id='organization-datasets-search-form', type=dataset_type, query=q, sorting=sorting, sorting_selected=sort_by_selected, count=page.item_count, facets=facets, placeholder=_('Search datasets...'), show_empty=request.args, fields=fields, force_title=_('Datasets') %}
{% endblock %}
{% block packages_list %}
{% if c.page.items %}
{{ h.snippet('snippets/package_list.html', packages=c.page.items) }}
{% if page.items %}
{{ h.snippet('snippets/package_list.html', packages=page.items) }}
{% endif %}
{% endblock %}
{% block page_pagination %}
{{ c.page.pager(q=c.q) }}
{{ page.pager(q=q) }}
{% endblock %}
{% endblock %}
{% block organization_facets %}
{% set url = h.url_for(c.group_dict.type ~ '_read', id=c.group_dict.id) %}
{% set url = h.url_for(group_dict.type ~ '_read', id=group_dict.id) %}
<div class="filters">
<div class="container-fluid filter-reset">
<div class="filter-reset-label"><span>{{ _('Filter') }}:</span></div>
<div class="filter-reset-box"><a href="{{ url }}">{{ _('Reset') }}</a></button></div>
</div>
<div>
{% for facet in c.facet_titles %}
{{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet,
extras={'id':c.group_dict.id}) }}
{% for facet in facet_titles %}
{{ h.snippet('snippets/facet_list.html', title=facet_titles[facet], name=facet, extras={'id':group_dict.id}, search_facets=search_facets) }}
{% endfor %}
</div>
<a class="close no-text hide-filters" aria-label="{{ _('Close') }}"><i class="fa fa-times-circle"></i><span class="text">{{ _('Close') }}</span></a>
......
ckanext/odsh/templates/package/search.html
+ 44
− 44
View file @ ed80c273
ckanext/odsh/templates/snippets/facet_list.html
+ 1
− 1
View file @ ed80c273
......@@ -63,7 +63,7 @@
<div class="module-footer">
{% if not name=='groups' %}
{% if h.get_param_int('_%s_limit' % name) %}
{% if h.odsh_has_more_facets(name) %}
{% if h.odsh_has_more_facets(name, search_facets) %}
<a href="{{ h.remove_url_param('_%s_limit' % name, replace=0, extras=extras, alternative_url=alternative_url) }}"
class="read-more">> {{ _('Show More') }}</a>
{% endif %}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

Impressum - Datenschutz - Barrierefreiheit