diff --git a/ckanext/odsh/collection/controller.py b/ckanext/odsh/collection/controller.py
index 5b049563195ed12b5a1b4f008b38fd95fc3af612..79b669afb67b03042be0c3c4b58cf5b5c1f63a0d 100644
--- a/ckanext/odsh/collection/controller.py
+++ b/ckanext/odsh/collection/controller.py
@@ -1,7 +1,7 @@
from ckan.lib.helpers import is_url, url_for
import ckan.plugins.toolkit as toolkit
from ckan.controllers.package import PackageController
-from helpers import get_latest_resources_for_type, get_latest_dataset
+from helpers import get_latest_resources_for_format, get_latest_dataset
@@ -13,10 +13,10 @@ class LatestDatasetController(PackageController):
class LatestRecourcesController(PackageController):
- def latest_resource(self, id, type):
- latest_resources = get_latest_resources_for_type(id, type)
+ def latest_resource(self, id, resource_format):
+ latest_resources = get_latest_resources_for_format(id, resource_format)
if latest_resources is None:
- abort(404)
+ toolkit.abort(404)
url_type = latest_resources.get('url_type')
if url_type is None:
resource_url = latest_resources.get('url')
@@ -35,4 +35,4 @@ class LatestRecourcesController(PackageController):
filename=pre_resource_url,
qualified = True)
toolkit.redirect_to(url_resource)
- abort(404)
\ No newline at end of file
+ toolkit.abort(404)
\ No newline at end of file
diff --git a/ckanext/odsh/collection/helpers.py b/ckanext/odsh/collection/helpers.py
index ec0f7dcbf05e7c37afa5491501e9d54f071f11d8..282fe5e0c1a0be68034caa18ba201181970f5291 100644
--- a/ckanext/odsh/collection/helpers.py
+++ b/ckanext/odsh/collection/helpers.py
@@ -1,161 +1,159 @@
from string import lower
+from operator import itemgetter
import ckan.lib.helpers as helpers
import ckan.model as model
import ckan.plugins.toolkit as toolkit
+def get_collection(dataset_dict):
+ collection_id = get_collection_id(dataset_dict)
+ if collection_id:
+ return get_collection_info(collection_id, dataset_dict)
+
+ return None
+
+
+def get_collection_info(collection_id, dataset_dict=None):
+ collection_dict = get_package_dict(collection_id)
+ dataset_names = get_dataset_names(collection_dict)
+ datasets_in_collection = get_datasets_from_solr(dataset_names)
+ collection_info = gather_collection_info(collection_dict, datasets_in_collection, dataset_dict)
+ return collection_info
+
+
+def get_collection_id(dataset_dict):
+ relationships_dataset = dataset_dict.get('relationships_as_subject')
+ if relationships_dataset and len(relationships_dataset):
+ return relationships_dataset[0]['__extras']['object_package_id']
+ relationships_dataset = dataset_dict.get('relationships')
+ if relationships_dataset and len(relationships_dataset):
+ return relationships_dataset[0].get('object')
+ return None
-#routine functions
def get_package_dict(name):
return model.Package.get(name).as_dict()
-def get_relationships(name):
- collection_dict = get_package_dict(name)
- return collection_dict.get('relationships')
-def get_all_datasets_belonging_to_collection(collection_name):
- list_rel_collection = get_relationships(collection_name)
- name_list = list()
- for item in list_rel_collection:
- item_object = item.get('object')
- name_list.append(item_object)
- return name_list
+def get_dataset_names(collection_dict):
+ collection_dict = get_package_dict(collection_dict.get('id')) # needed to get full package_dict
+ relationships_collection = collection_dict.get('relationships')
+ names_collection_members = [relationship.get('object') for relationship in relationships_collection]
+ return names_collection_members
-#for mapping latest resources and latest dataset
-def get_latest_dataset(collection_name):
- collection_list_relationships = get_all_datasets_belonging_to_collection(collection_name)
- latest_issued = latest_name = None
- for item in collection_list_relationships:
- item_pkt_dict = get_package_dict(item)
- if helpers.check_access('package_show', item_pkt_dict):
- item_issued = item_pkt_dict.get('extras').get('issued')
- if latest_issued < item_issued or (latest_issued == item_issued and latest_name < item):
- latest_name=item
- latest_issued=item_issued
- return latest_name
+def get_datasets_from_solr(dataset_names):
+ context = None
+ name_expression = ' OR '.join(dataset_names)
+ fq = 'name:({})'.format(name_expression)
+
+ sort = 'name asc,extras_issued asc'
+
+ # maximum possible number of results is 1000,
+ # see https://docs.ckan.org/en/ckan-2.7.3/api/index.html#ckan.logic.action.get.package_search
+ query_result = toolkit.get_action('package_search')(context, {
+ 'fq': fq,
+ 'sort': sort,
+ 'rows': 1000,
+ })
-def is_latest_resources(resource_format, type, resource_created,latest_created, resource_id, latest_id):
- if lower(resource_format) == lower(type):
- return (resource_created > latest_created or (resource_created == latest_created and resource_id > latest_id))
- else:
- return False
-
-def get_latest_resources_for_type(collection_name, type):
- latest_dataset_name = get_latest_dataset(collection_name)
- latest_dataset = get_package_dict(latest_dataset_name)
- resource_list = latest_dataset.get('resources')
- latest_resource = latest_created = latest_id = None
- for resource in resource_list:
- resource_format = resource.get('format')
- resource_created = resource.get('created')
- resource_id = resource.get('id')
- if is_latest_resources(resource_format, type, resource_created, latest_created, resource_id, latest_id):
- latest_id=resource_id
- latest_created=resource_created
- latest_resource=resource
- return latest_resource
-
-#for predecessor and successor
-
-
-
-def get_collection_name_by_dataset(dataset_name):
- list_rel_dataset = get_relationships(dataset_name)
- if len(list_rel_dataset):
- return list_rel_dataset[0]['object']
-
-def get_collection_title_by_dataset(pkg_dict_dataset):
- dataset_name = pkg_dict_dataset.get('name')
- collection_name = get_collection_name_by_dataset(dataset_name)
- if not collection_name:
- return None
- context = None
- pkg_dict_collection = toolkit.get_action('package_show')(context, {'id': collection_name})
- if not pkg_dict_collection:
- return None
- title_collection = pkg_dict_collection.get('title')
- return title_collection
-
-def get_all_datasets_belonging_to_collection_by_dataset(dataset_name):
- collection_name = get_collection_name_by_dataset(dataset_name)
- if collection_name:
- return get_all_datasets_belonging_to_collection(collection_name)
- return []
-
-def _get_siblings_dicts_with_access(pkg_dict):
- dataset_name = pkg_dict.get('name')
- list_of_siblings = get_all_datasets_belonging_to_collection_by_dataset(dataset_name)
- n_siblings = len(list_of_siblings)
- if n_siblings>0:
- siblings_dicts = [get_package_dict(name) for name in list_of_siblings]
- user_has_access = lambda pkg_dict:helpers.check_access('package_show', pkg_dict)
- siblings_dicts_with_access = filter(user_has_access, siblings_dicts)
- return siblings_dicts_with_access
- return None
+ results = query_result.get('results')
+ datasets_found = results if results else []
+
+ return datasets_found
-def _sort_siblings_by_name_and_date(siblings_dicts):
- '''
- sort by name first and then by date to have a fallback if dates are the same
- '''
- _get_name = lambda pkg_dict:pkg_dict.get('name')
- _get_issued = lambda pkg_dict:pkg_dict.get('extras').get('issued')
- siblings_dicts_sorted_by_name = sorted(siblings_dicts, key=_get_name)
- siblings_dicts_sorted_by_date_issued = sorted(siblings_dicts_sorted_by_name, key=_get_issued)
- return siblings_dicts_sorted_by_date_issued
-
-def get_successor_and_predecessor_dataset(pkg_dict):
- dataset_name = pkg_dict.get('name')
- siblings_dicts_with_access = _get_siblings_dicts_with_access(pkg_dict)
- if siblings_dicts_with_access:
- n_siblings = len(siblings_dicts_with_access)
- siblings_dicts_sorted_by_date_issued = _sort_siblings_by_name_and_date(siblings_dicts_with_access)
- siblings_names_sorted_by_date_issued = [d['name'] for d in siblings_dicts_sorted_by_date_issued]
- id_current_dataset = siblings_names_sorted_by_date_issued.index(dataset_name)
- predecessor_name = (
- siblings_names_sorted_by_date_issued[id_current_dataset-1] if (id_current_dataset > 0)
- else None
- )
- successor_name = (
- siblings_names_sorted_by_date_issued[id_current_dataset+1] if (id_current_dataset < n_siblings-1)
- else None
- )
- else:
- predecessor_name, successor_name = None, None
- return successor_name, predecessor_name
-
-def get_successor_and_predecessor_urls(pkg_dict):
- successor_name, predecessor_name = get_successor_and_predecessor_dataset(pkg_dict)
- successor_url, predecessor_url = (
- helpers.url_for(controller='package', action='read', id=name)
- if name is not None
- else None
- for name in (successor_name, predecessor_name)
- )
- return successor_url, predecessor_url
-def get_successor(pkg_dict):
- successor_and_predecessor = get_successor_and_predecessor_urls(pkg_dict)
- return successor_and_predecessor[0]
+def gather_collection_info(collection_dict, datasets_in_collection, dataset_dict=None):
+ name_first_dataset = datasets_in_collection[0].get('name')
+ url_first_dataset = url_from_id(name_first_dataset)
-def get_predecessor(pkg_dict):
- successor_and_predecessor = get_successor_and_predecessor_urls(pkg_dict)
- return successor_and_predecessor[1]
-
-#link to latest collection member
-def latest_collection_member_persistent_link(pkg_dict):
- dataset_name = pkg_dict.get('name')
- collection_name = get_collection_name_by_dataset(
- dataset_name=dataset_name
- )
- if not collection_name:
- return None
- url = helpers.url_for(
+ name_last_dataset = datasets_in_collection[-1].get('name')
+ url_last_dataset = url_from_id(name_last_dataset)
+
+ name_collection = collection_dict.get('name')
+ persistent_link_last_member = url_last_member(name_collection)
+
+ url_collection = url_from_id(collection_dict.get('id'))
+
+ if dataset_dict:
+ name_current_dataset = dataset_dict.get('name')
+ dataset_names = [d.get('name') for d in datasets_in_collection]
+
+ def get_predecessor():
+ try:
+ id_current = dataset_names.index(name_current_dataset)
+ except ValueError:
+ return None
+ if id_current > 0:
+ return dataset_names[id_current - 1]
+ return None
+
+ def get_successor():
+ try:
+ id_current = dataset_names.index(name_current_dataset)
+ except ValueError:
+ return None
+ if id_current < len(dataset_names) - 1:
+ return dataset_names[id_current + 1]
+ return None
+
+ name_predecessor = get_predecessor()
+ url_predecessor = url_from_id(name_predecessor) if name_predecessor else None
+
+ name_successor = get_successor()
+ url_successor = url_from_id(name_successor) if name_successor else None
+ else:
+ url_predecessor = url_successor = None
+
+ return {
+ 'title': collection_dict.get('title'),
+ 'url': url_collection,
+ 'members': datasets_in_collection,
+ 'first_member': {
+ 'name': name_first_dataset,
+ 'url': url_first_dataset,
+ },
+ 'last_member': {
+ 'name': name_last_dataset,
+ 'url': url_last_dataset,
+ },
+ 'predecessor': {
+ 'url': url_predecessor,
+ },
+ 'successor': {
+ 'url': url_successor,
+ },
+ 'persistent_link_last_member': persistent_link_last_member,
+ }
+
+def url_from_id(package_id):
+ return helpers.url_for(controller='package', action='read', id=package_id)
+
+def url_last_member(name_collection):
+ return helpers.url_for(
controller='ckanext.odsh.collection.controller:LatestDatasetController',
action='latest',
- id=collection_name
+ id=name_collection
)
- return url
\ No newline at end of file
+
+
+def get_latest_dataset(collection_name):
+ collection_info = get_collection_info(collection_name)
+ latest_name = collection_info['last_member']['name']
+ return latest_name
+
+
+def get_latest_resources_for_format(collection_name, resource_format):
+ collection_info = get_collection_info(collection_name)
+ members = collection_info.get('members')
+ if not members:
+ return None
+ latest_dataset = members[-1]
+ resources = latest_dataset.get('resources')
+ if not resources:
+ return None
+ resources_with_asked_type = [r for r in resources if r.get('format').upper() == resource_format.upper()]
+ resources_sorted = sorted(resources_with_asked_type, key=itemgetter('id','created'), reverse=True)
+ return resources_sorted[-1]
diff --git a/ckanext/odsh/collection/plugin.py b/ckanext/odsh/collection/plugin.py
index 2ebcefc1af2c164045af7e861856858644885df9..27a28eccf53d86da24b79a31732bf10c37dc8ff5 100644
--- a/ckanext/odsh/collection/plugin.py
+++ b/ckanext/odsh/collection/plugin.py
@@ -1,6 +1,7 @@
from ckan.lib.plugins import DefaultTranslation, DefaultDatasetForm
-import ckan.plugins as plugins
+import ckan.plugins as plugins
+import ckan.plugins.toolkit as toolkit
import helpers as collection_helpers
from routes.mapper import SubMapper
@@ -16,6 +17,9 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
def is_fallback(self):
return False
+
+ def read_template(self):
+ return 'package/collection_read.html'
# IRoutes
@@ -28,7 +32,7 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
)
map.connect(
- '/collection/{id}/aktuell.{type}',
+ '/collection/{id}/aktuell.{resource_format}',
controller='ckanext.odsh.collection.controller:LatestRecourcesController',
action='latest_resource'
)
@@ -39,7 +43,7 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
path_prefix='/collection/'
) as m:
m.connect('latest', '{id}/aktuell', action='latest')
- m.connect('latest_resource', '{id}/aktuell.{type}', action='latest_resource')
+ m.connect('latest_resource', '{id}/aktuell.{resource_format}', action='latest_resource')
return map
def after_map(self, map):
@@ -53,8 +57,8 @@ class CollectionsPlugin(plugins.SingletonPlugin, DefaultDatasetForm):
# other extensions.
return {
- 'collection_get_successor': collection_helpers.get_successor,
- 'collection_get_predecessor': collection_helpers.get_predecessor,
- 'collection_get_latest_member':collection_helpers.latest_collection_member_persistent_link,
- 'collection_get_title': collection_helpers.get_collection_title_by_dataset,
+ 'get_collection': collection_helpers.get_collection,
+ 'get_collection_info': collection_helpers.get_collection_info,
+ 'url_from_id': collection_helpers.url_from_id,
}
+
\ No newline at end of file
diff --git a/ckanext/odsh/controller.py b/ckanext/odsh/controller.py
index 77076573f7570ef28089d1519ecb61121cdd964f..daf5d9efba32a5dbb132ca4877dcee7fa1464d01 100644
--- a/ckanext/odsh/controller.py
+++ b/ckanext/odsh/controller.py
@@ -11,7 +11,6 @@ from ckan.controllers.package import PackageController
from ckan.controllers.feed import FeedController, ITEMS_LIMIT, _package_search, _create_atom_id
import ckan.lib.helpers as h
import ckan.authz as authz
-from ckan.common import c
import logging
import matomo
import ckan.logic as logic
@@ -21,6 +20,8 @@ import ckan.plugins.toolkit as toolkit
from ckanext.dcat.controllers import DCATController
import ckan.model as model
import helpers
+import json
+
abort = base.abort
log = logging.getLogger(__name__)
@@ -30,10 +31,10 @@ get_action = logic.get_action
class OdshRouteController(HomeController):
def info_page(self):
- h.redirect_to('http://www.schleswig-holstein.de/odpinfo')
+ h.redirect_to('http://www.schleswig-holstein.de/tpinfo')
def start(self):
- h.redirect_to('http://www.schleswig-holstein.de/odpstart')
+ h.redirect_to('http://www.schleswig-holstein.de/tpstart')
def not_found(self):
abort(404)
@@ -143,6 +144,12 @@ class OdshGroupController(OrganizationController):
class OdshApiController(ApiController):
def action(self, logic_function, ver=None):
+ if toolkit.asbool(config.get('ckanext.odsh.log_api_requests', 'false')):
+ try:
+ request_data = self._get_request_data(try_url_params=False)
+ log.info('POST request body: {}'.format(request_data))
+ except Exception, e:
+ log.error(e)
if logic_function == 'resource_qv4yAI2rgotamXGk98gJ':
return helpers.odsh_get_version_id()
if logic_function == 'resourcelog_qv4yAI2rgotamXGk98gJ':
diff --git a/ckanext/odsh/helper_pkg_dict.py b/ckanext/odsh/helper_pkg_dict.py
index 912b0b83db873b0ed58b98e64ef21b83c3a59e4b..487a8ff5c89933829f2711203c08171df1a31f4a 100644
--- a/ckanext/odsh/helper_pkg_dict.py
+++ b/ckanext/odsh/helper_pkg_dict.py
@@ -102,18 +102,6 @@ class HelperPgkDict(object):
return None
- def get_collection_id(self):
- '''
- construct a collection uri from the id of
- the containing collection
- '''
- package_name = self.pkg_dict.get('name')
- collection_name = helpers_collection.get_collection_name_by_dataset(package_name)
- collection_dict = helpers_collection.get_package_dict(collection_name)
- collection_id = collection_dict.get('id')
- return collection_id
-
-
def add_uri_to_store(self):
'''
store pair of uri and id for subsequent use
diff --git a/ckanext/odsh/helpers_tpsh.py b/ckanext/odsh/helpers_tpsh.py
index 608c91016c353ca9de29934c3b06a0c52dd1c4d8..927313ed03fa9623f92f4f7e2018ed425c918891 100644
--- a/ckanext/odsh/helpers_tpsh.py
+++ b/ckanext/odsh/helpers_tpsh.py
@@ -8,6 +8,8 @@ import json
import re
import urllib2
from collections import OrderedDict
+import subprocess
+import os
from ckan.common import config
import ckan.lib.helpers as helpers
@@ -170,7 +172,11 @@ def get_language_for_selection():
return dict_for_select_box
def get_package_dict(name):
- return model.Package.get(name).as_dict()
+ '''
+ raises ckan.logic.NotFound if not found
+ '''
+ package_dict = toolkit.get_action('package_show')(None, {'id': name})
+ return package_dict
def size_of_fmt(num, suffix='B'):
for unit in ['',' k',' M',' G',' T',' P',' E',' Z']:
@@ -208,4 +214,12 @@ def get_body_mail(organization, package):
mail_document = "Dokument-ID: " + package_name + "%0D%0A"
mail_url = "URL: " + url + "%0D%0A" + "%0D%0A"
message = mail_titel + mail_document + mail_url + "Mein Kommentar:" + "%0D%0A" + "%0D%0A" + "%0D%0A" + "%0D%0A"
- return anrede + message
\ No newline at end of file
+ return anrede + message
+
+def git_commit_hash():
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ try:
+ command = 'git log -n 1 --format=%H'
+ except:
+ return 'unknown'
+ return subprocess.check_output([command], shell=True, cwd=current_dir)
\ No newline at end of file
diff --git a/ckanext/odsh/logic/action.py b/ckanext/odsh/logic/action.py
index 481a72ac130bf8447c89be96dcc27f6ddde28832..0b4402dbcc986b67f8c91964d4c15820785c0172 100644
--- a/ckanext/odsh/logic/action.py
+++ b/ckanext/odsh/logic/action.py
@@ -1,34 +1,38 @@
-import logging
import ckan.logic as logic
from ckan.logic.action.update import user_update
-from ckan.logic.action.create import package_create, user_create, group_member_create
+from ckan.logic.action.create import package_create, resource_create, user_create, group_member_create
import ckan.model as model
import ckan.lib.dictization.model_dictize as model_dictize
from ckan.lib.munge import munge_title_to_name
import ckan.plugins.toolkit as toolkit
-from ckan.lib.search.common import (
- make_connection, SearchError, SearchQueryError
-)
+from ckan.lib.search.common import make_connection, SearchError
import pysolr
import datetime
+import cgi
+import urllib2
+import logging
log = logging.getLogger(__name__)
+from ckanext.odsh.setup_proxy import setup_proxy, clear_proxy
+from ckanext.odsh.collection.helpers import get_collection_id, get_package_dict
+from ckanext.odsh.helpers_tpsh import add_pkg_to_collection
+from ckanext.odsh.helpers import odsh_extract_value_from_extras
+
def odsh_package_create(context, data_dict):
pkg_type = data_dict.get('type', None)
if pkg_type == 'collection':
return package_create(context, data_dict)
munge_increment_name(data_dict)
+ related_package_name = data_dict.get('relatedPackage')
+ if related_package_name:
+ new_package = package_create(context, data_dict)
+ add_to_same_collection(related_package_name, new_package['name'])
+ return new_package
if pkg_type != 'dataset':
return package_create(context, data_dict)
- issued = False
- for extra in data_dict.get('extras'):
- if extra['key'] == 'issued':
- issued = True
- break
- if not issued:
- data_dict['extras'].append({'key': 'issued', 'value': datetime.datetime.utcnow().isoformat()})
+ add_issued_if_missing(data_dict)
return package_create(context, data_dict)
@@ -45,6 +49,43 @@ def munge_increment_name(data_dict):
log.debug('name: %s' % name)
data_dict['name'] = name
+
+def add_issued_if_missing(data_dict):
+ issued = odsh_extract_value_from_extras(data_dict.get('extras'), 'issued')
+ if issued is None:
+ data_dict['extras'].append({'key': 'issued', 'value': datetime.datetime.utcnow().isoformat()})
+
+
+def add_to_same_collection(related_package_name, package_name):
+ related_package = get_package_dict(related_package_name)
+ collection_id = get_collection_id(related_package)
+ if not collection_id:
+ collection_dict = auto_create_collection(related_package)
+ collection_id = collection_dict.get('id')
+ add_to_collection(collection_id, related_package_name)
+ add_to_collection(collection_id, package_name)
+
+
+def auto_create_collection(related_package):
+ related_package_title = related_package.get('title')
+ owner_org = related_package.get('owner_org')
+ _collection_dict = {
+ 'title': related_package_title,
+ 'owner_org': owner_org,
+ 'type': 'collection',
+ }
+ munge_increment_name(_collection_dict)
+ collection_dict = package_create_via_toolkit(None, _collection_dict)
+ return collection_dict
+
+
+def package_create_via_toolkit(context, package_dict):
+ return toolkit.get_action('package_create')(context, package_dict)
+
+
+def add_to_collection(collection_id, package_name):
+ add_pkg_to_collection(package_name, collection_id)
+
def check_password(password):
return (len(password) >= 8 and
@@ -52,8 +93,10 @@ def check_password(password):
any(c.isupper() for c in password) and
any((c.isalpha()==False) for c in password)) #Number or Special character
+
PASSWORD_ERROR_MESSAGE = {'security': ['Passwort muss mindestens acht Zeichen, einen Gross-, einen Kleinbuchstaben und entweder eine Zahl oder ein Sondernzeichen enthalten!']}
+
def odsh_user_create(context, data_dict):
model = context['model']
password = data_dict.get('password')
@@ -69,6 +112,7 @@ def odsh_user_create(context, data_dict):
else:
raise logic.ValidationError(PASSWORD_ERROR_MESSAGE)
+
def tpsh_user_update(context, data_dict):
password = data_dict.get('password')
if not password:
@@ -78,8 +122,6 @@ def tpsh_user_update(context, data_dict):
return user_update(context, data_dict)
-
-
@toolkit.side_effect_free
def autocomplete(context, data_dict):
query = {
@@ -102,3 +144,41 @@ def autocomplete(context, data_dict):
filtered_suggestions.append(suggestion)
final_suggestions = list(sorted(set(filtered_suggestions), key=filtered_suggestions.index))[:5]
return final_suggestions
+
+
+def odsh_resource_create(context, data_dict):
+ is_linked_resource = not isinstance(data_dict['upload'], cgi.FieldStorage)
+ if is_linked_resource:
+ _download_linked_resource_to_tmp(data_dict['url'])
+ _emulate_file_upload(data_dict)
+ return resource_create(context, data_dict)
+
+TMP_FILE_PATH = '/tmp/temp_file_upload'
+
+def _download_linked_resource_to_tmp(url):
+ log.debug('Downloading linked resource from {}.'.format(url))
+ setup_proxy()
+ test_file = urllib2.urlopen(url).read()
+ clear_proxy()
+ with open(TMP_FILE_PATH, 'wb') as temporary_file:
+ temporary_file.write(test_file)
+
+def _emulate_file_upload(data_dict):
+ '''
+ This function updates the data_dict in order to emulate
+ the behaviour of resource creation with uploaded file for
+ resource creation with link
+ '''
+ temporary_file = open(TMP_FILE_PATH, 'rb')
+ upload_file = temporary_file
+ filename = data_dict['name']
+ upload = cgi.FieldStorage()
+ upload.disposition = 'form-data'
+ upload.disposition_options = {'filename': filename, 'name': 'upload'}
+ upload.fp = upload_file
+ upload.file = upload_file
+ upload.type = 'application/pdf'
+ upload.headers['content-type'] = upload.type
+ upload.name = 'upload'
+ upload.filename = filename
+ data_dict['upload'] = upload
\ No newline at end of file
diff --git a/ckanext/odsh/plugin.py b/ckanext/odsh/plugin.py
index 708406df3d57203657d9201b4ba61afdb951d44a..076ca28f029e4798b4b5ba87dc62c3b368ac6aa0 100644
--- a/ckanext/odsh/plugin.py
+++ b/ckanext/odsh/plugin.py
@@ -51,7 +51,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
def get_actions(self):
return {'package_create': action.odsh_package_create,
'user_update':action.tpsh_user_update,
- 'user_create': action.odsh_user_create}
+ 'user_create': action.odsh_user_create,
+ 'resource_create': action.odsh_resource_create,}
# IConfigurer
@@ -127,7 +128,14 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
-
+ 'relatedPackage': [
+ toolkit.get_validator('tpsh_validate_relatedPackage'),
+ toolkit.get_converter('convert_to_extras')
+ ],
+ 'accessibility': [
+ toolkit.get_validator('ignore_missing'),
+ toolkit.get_converter('convert_to_extras')
+ ]
})
return schema
@@ -141,6 +149,14 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
toolkit.get_converter('convert_from_extras'),
toolkit.get_validator('ignore_missing')
],
+ 'relatedPackage': [
+ toolkit.get_converter('convert_from_extras'),
+ toolkit.get_validator('ignore_missing')
+ ],
+ 'accessibility': [
+ toolkit.get_converter('convert_from_extras'),
+ toolkit.get_validator('ignore_missing')
+ ],
})
return schema
@@ -320,7 +336,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
# Template helper function names should begin with the name of the
# extension they belong to, to avoid clashing with functions from
# other extensions.
- return {'odsh_main_groups': odsh_helpers.odsh_main_groups,
+ return {
+ 'odsh_main_groups': odsh_helpers.odsh_main_groups,
'odsh_now': odsh_helpers.odsh_now,
'odsh_group_id_selected': odsh_helpers.odsh_group_id_selected,
'odsh_get_facet_items_dict': odsh_helpers.odsh_get_facet_items_dict,
@@ -357,7 +374,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
'tpsh_get_resource_size': helpers_tpsh.get_resource_size,
'tpsh_get_address_org':helpers_tpsh.get_address_org,
'tpsh_get_body_mail':helpers_tpsh.get_body_mail,
- }
+ 'tpsh_git_commit_hash': helpers_tpsh.git_commit_hash,
+ }
# IValidators
diff --git a/ckanext/odsh/profiles/odsh_dcat_de_profile.py b/ckanext/odsh/profiles/odsh_dcat_de_profile.py
index c475ac6a823c6bac61712051a89fecae077cdcd3..67b277164df81864524f7018811323557b7f2ed7 100644
--- a/ckanext/odsh/profiles/odsh_dcat_de_profile.py
+++ b/ckanext/odsh/profiles/odsh_dcat_de_profile.py
@@ -10,7 +10,6 @@ from ckanext.dcatde.profiles import DCATdeProfile, DCATDE, DCAT, DCATDE_1_0
import ckanext.odsh.helpers_tpsh as helpers_tpsh
import ckanext.odsh.collection.helpers as helpers_collection
-from ckanext.odsh.helper_pkg_dict import HelperPgkDict
DCT = rdflib.namespace.Namespace("http://purl.org/dc/terms/")
@@ -134,10 +133,8 @@ class ODSHDCATdeProfile(DCATdeProfile):
return is_collection
def _get_dataset_refs_belonging_to_collection(self, dataset_dict):
- dataset_names = helpers_collection.get_all_datasets_belonging_to_collection(
- collection_name = dataset_dict.get('id')
- )
- dataset_dicts = [model.Package.get(name).as_dict() for name in dataset_names]
+ dataset_names = helpers_collection.get_dataset_names(dataset_dict)
+ dataset_dicts = [model.Package.get(name).as_dict() for name in dataset_names]
dataset_ids = [dataset_dict.get('id') for dataset_dict in dataset_dicts]
dataset_refs = [self._construct_refs(id) for id in dataset_ids]
return dataset_refs
@@ -155,12 +152,11 @@ class ODSHDCATdeProfile(DCATdeProfile):
'''
if dataset_dict.get('type')=='collection':
return False
- id_dataset = dataset_dict.get('id')
- collection_name = helpers_collection.get_collection_name_by_dataset(id_dataset)
- return collection_name is not None
+ collection_name = helpers_collection.get_collection_id(dataset_dict)
+ return collection_name is not None
def _add_collection(self, dataset_dict, dataset_ref):
- collection_id = HelperPgkDict(dataset_dict).get_collection_id()
+ collection_id = helpers_collection.get_collection_id(dataset_dict)
collection_uri = self._construct_refs(collection_id)
self.g.set(
(dataset_ref, DCT.isVersionOf,
diff --git a/ckanext/odsh/public/odsh.css b/ckanext/odsh/public/odsh.css
index 516b49214855d61fa81b240ee82012eb547ca866..a51d97ae0b603cbb74abc58c4fe14df00da8acc2 100644
--- a/ckanext/odsh/public/odsh.css
+++ b/ckanext/odsh/public/odsh.css
@@ -130,6 +130,11 @@ h3{
}
}
+ .site-collection-summary .odsh-dataset-heading {
+ margin-left: 0px;
+ margin-right: 0px;
+ }
+
.search-form, .odsh-dataset-heading {
display: flex;
align-items: center;
@@ -402,6 +407,11 @@ input[type=radio], input[type=checkbox] {
max-width: 300px;
padding: 40.66px 16px 0px;
}
+
+.site-collection-summary .secondary.span3 {
+ display: none;
+}
+
.secondary .module-narrow .module-content{
padding-left: 0px;
padding-right: 0px;
@@ -411,6 +421,10 @@ input[type=radio], input[type=checkbox] {
padding-left: 31px;
}
+.site-collection-summary .primary {
+ padding-left: 0;
+}
+
.primary.organization {
padding-left: 0px;
}
@@ -1270,6 +1284,7 @@ body {
.footer-icon{
float: left;
margin-right: 40px;
+ margin-bottom: 1rem;
}
.footer-icon.last{
@@ -1314,7 +1329,7 @@ body {
box-sizing: border-box;
padding-bottom: 15px;
background-image: none;
- height: 139px;
+ height: 150px;
display: flex;
flex-direction: column-reverse;
}
@@ -1453,6 +1468,24 @@ body {
color: #000000;
}
++.hint-newer-version {
+ margin-top: 30px;
+ margin-bottom: 30px;
+ font-size: 20px;
+ color: white;
+ background-color: #D4004B;
+ padding: 0.5rem 1rem;
+}
+
+.hint-newer-version > a {
+ color: white;
+ text-decoration: underline;
+}
+
+.hint-newer-version > a:hover {
+ color: lightgray;
+}
+
.resource-list {
margin: 0px;
}
@@ -2770,3 +2803,7 @@ body.filters-modal div.row > aside.secondary.span3 {
margin-right: 6px;
}
}
+
+.tpsh-collection-list {
+ list-style-type: none;
+}
diff --git a/ckanext/odsh/public/odsh_header.css b/ckanext/odsh/public/odsh_header.css
index 31822582d5b095bb524b579579b44830b182786a..11a784cb416347446d822aff454e94be5102fbce 100644
--- a/ckanext/odsh/public/odsh_header.css
+++ b/ckanext/odsh/public/odsh_header.css
@@ -242,7 +242,7 @@
}
@media (min-width: 768px) {
- .navbar.masthead .navigation .nav-pills li:not(:first-child) {
+ .navbar.masthead .navigation .nav-pills li:not(:nth-child(2)) {
margin-left: 22px;
}
}
diff --git a/ckanext/odsh/setup_proxy.py b/ckanext/odsh/setup_proxy.py
new file mode 100644
index 0000000000000000000000000000000000000000..13338f65a60156bf8958107746035d50d8e2c9ab
--- /dev/null
+++ b/ckanext/odsh/setup_proxy.py
@@ -0,0 +1,24 @@
+import urllib2
+from ckan.common import config
+
+
+def setup_proxy():
+ '''
+ This function declares that a proxy server shall be used to access the web via
+ urllib2. It takes the proxy address from ckan's config file
+ (
+ field "ckanext.odsh.download_proxy",
+ example: ckanext.odsh.download_proxy = http://1.2.3.4:4123
+ )
+ '''
+
+ proxy_url = config.get('ckanext.odsh.download_proxy', None)
+ if proxy_url:
+ proxy = urllib2.ProxyHandler({'http': proxy_url, 'https': proxy_url})
+ opener = urllib2.build_opener(proxy)
+ urllib2.install_opener(opener)
+
+def clear_proxy():
+ proxy = urllib2.ProxyHandler({})
+ opener = urllib2.build_opener(proxy)
+ urllib2.install_opener(opener)
\ No newline at end of file
diff --git a/ckanext/odsh/templates/footer.html b/ckanext/odsh/templates/footer.html
index 03764a7c7b58782fb85faee0af0af365732205df..1f41aab6d42ee6f8b6bf70970bb1162fd14f2198 100644
--- a/ckanext/odsh/templates/footer.html
+++ b/ckanext/odsh/templates/footer.html
@@ -12,8 +12,9 @@
<div class="footer-right">
<div class='footer-icon'><a href='http://www.schleswig-holstein.de/tpkontakt'>Kontakt</a></div>
<div class='footer-icon'><a href='http://www.schleswig-holstein.de/tpimpressum'>Impressum</a></div>
+ <div class='footer-icon'><a href='https://www.schleswig-holstein.de/DE/Serviceseiten/Barrierefreiheitserklaerung/barrierefreiheit_node.html'>Barrierefreiheit</a></div>
<div class='footer-icon last'><a href='http://www.schleswig-holstein.de/tpdatenschutz'>Datenschutz</a></div>
</div>
</div>
</div>
-</footer>
\ No newline at end of file
+</footer>
diff --git a/ckanext/odsh/templates/package/collection_read.html b/ckanext/odsh/templates/package/collection_read.html
new file mode 100644
index 0000000000000000000000000000000000000000..c76cb3be210518bb7c05cf6eb9df3aaf4f9edcd8
--- /dev/null
+++ b/ckanext/odsh/templates/package/collection_read.html
@@ -0,0 +1,25 @@
+{% extends "package/read.html" %}
+
+{% block bodytag %} {{ super() }} class="site-collection-summary" {% endblock %}
+
+{% block package_resources %}
+{% endblock package_resources %}
+
+{% block secondary_content %}
+{% endblock secondary_content %}
+
+{% block collection %}
+
+{% set collection_id = pkg.get('id') %}
+{% set collection = h.get_collection_info(collection_id) %}
+{% set collection_title = collection['title'] if collection else None %}
+
+{% if collection['members'] %}
+<p>
+ Dies ist die Übersichtsseite der Dokumentensammlung {{ collection_title }}.
+ Sie enthält folgende Dokumente (neuere Dokumente zuerst):
+</p>
+{% snippet 'snippets/package_list.html', packages=collection['members'] | reverse() %}
+{% endif %}
+
+{% endblock collection %}
diff --git a/ckanext/odsh/templates/package/read.html b/ckanext/odsh/templates/package/read.html
index 068b60870df00513f3c63b30f848ad3724c0b7f6..860842a9398a0adf9a2c52decd4aea579b4b27b7 100644
--- a/ckanext/odsh/templates/package/read.html
+++ b/ckanext/odsh/templates/package/read.html
@@ -1,10 +1,12 @@
{% extends "package/read_base.html" %}
{% set pkg = c.pkg_dict %}
-{% set collection_title = h.collection_get_title(pkg) %}
-{% set successor_url = h.collection_get_successor(pkg) %}
-{% set predecessor_url = h.collection_get_predecessor(pkg) %}
-{% set latest_collection_member = h.collection_get_latest_member(pkg) %}
+{% set collection = h.get_collection(pkg) %}
+{% set collection_title = collection['title'] if collection else None %}
+{% set collection_url = collection['url'] if collection else None %}
+{% set successor_url = collection['successor']['url'] if collection else None %}
+{% set predecessor_url = collection['predecessor']['url'] if collection else None %}
+{% set latest_collection_member = collection['persistent_link_last_member'] if collection else None %}
{% set stars = h.odsh_openness_score_dataset_html(pkg) %}
{% block breadcrumb_content %}
@@ -22,7 +24,7 @@
<li>{% link_for _('Datasets'), controller='package', action='search' %}</li>
<li class="active"><a href="">{{ _('Create Dataset') }}</a></li>
{% endif %}
-{% endblock %}
+{% endblock breadcrumb_content %}
{% block primary_content_inner %}
{{ super() }}
@@ -37,7 +39,7 @@
{% if pkg.state == 'deleted' %}
[{{ _('Deleted') }}]
{% endif %}
- {% endblock %}
+ {% endblock page_heading%%}
<div class="odsh-dataset-edit-button">
{% if h.check_access('package_update', {'id':pkg.id }) %}
<div>
@@ -68,6 +70,12 @@
<img src="/base/images/icon_info.svg" aria-hidden="true"></img>
{{ _('Detailinformationen') }}
</div>
+{% if successor_url %}
+<p class="hint-newer-version">
+ Hinweis: Es ist eine <a href="{{ latest_collection_member }}">
+ neuere Version</a> dieses Dokuments verf�gbar.
+</p>
+{% endif %}
{% if pkg.notes %}
<div class="notes embedded-content">
{{ h.render_markdown(h.get_translated(pkg, 'notes')) }}
@@ -83,14 +91,14 @@
{% link_for _('Add new resource'), controller='package', action='new_resource', id=c.pkg_dict.name, class_='btn btn-primary', icon='plus' %}
</div>
{% endif %}
-{% endblock %}
+{% endblock package_resources %}
{% block collection %}
{% if latest_collection_member %}
<section id="dataset-collection">
{% if collection_title %}
<p>
- Dieser Datensatz ist Bestandteil der Datenreihe "{{ collection_title }}".
+ Dieses Dokument ist Bestandteil der <a href={{ collection_url }}>Dokumentengruppe "{{ collection_title }}"</a>.
Sie können zu älteren und neueren Datensätzen blättern.
</p>
{% endif %} {# collection_title #}
@@ -135,5 +143,5 @@
{% endfor %}
</div>
-{% endblock %}
+{% endblock primary_content_inner %}
diff --git a/ckanext/odsh/templates/snippets/package_item.html b/ckanext/odsh/templates/snippets/package_item.html
index b0918d3e68682688b5e93514aa556d969b320f13..6448db8f4582829b7620110bda14e20f865c414c 100644
--- a/ckanext/odsh/templates/snippets/package_item.html
+++ b/ckanext/odsh/templates/snippets/package_item.html
@@ -22,10 +22,10 @@ Example:
{% set license_name=' ('+licenseAttributionByText +')' if licenseAttributionByText else ''%}
{% set license = license_title + license_name %}
{% set org = package.organization.title or package.organization.name or '-' %}
-{% set stars = h.odsh_openness_score_dataset_html(package) %}
{% set access_count=(package.tracking_summary.total if package.tracking_summary) or '-' %}
{% set issued_extra = h.odsh_extract_value_from_extras(package.extras,'issued') %}
{% set issued = h.odsh_render_datetime(issued_extra) if issued_extra else h.odsh_render_datetime(package.metadata_created)%}
+{% set subject_text = h.odsh_extract_value_from_extras(package.extras,'subject_text') if h.odsh_extract_value_from_extras(package.extras,'subject_text') else '-' %}
{% set daterange = h.tpsh_get_daterange_prettified(package) %}
{% set language_of_package = h.tpsh_get_language_of_package(package) %}
{% set language_icon = h.get_language_icon(package) %}
@@ -33,6 +33,18 @@ Example:
{% block package_item %}
<div class="odsh-dataset-item">
+ <div class="preview-image-container">
+ {% if thumbnail %}
+ <a href={{ h.url_for(controller='package', action='read', id=package.name) }}>
+ <img src= "/{{ thumbnail }}" alt= "Vorschau" />
+ </a>
+
+ {% else %}
+ <a href={{ h.url_for(controller='package', action='read', id=package.name) }}>
+ <img src="/base/images/platzhalter.svg" alt= "keine Vorschau verfügbar" />
+ </a>
+ {% endif%}
+ </div>
{% block content %}
<div class="dataset-content">
{% block heading %}
@@ -107,40 +119,10 @@ Example:
{% endfor %}
</p>
{% endif %}
+ <p class='package-info-subject'>{{ _('subject') }}: {{ subject_text }} </p>
+ <p class='package-info-issued'>{{ _('issued') }}: {{issued}} </p>
{% endblock notes %}
</div>
-
- <div class='dataset-spacer'></div>
-
- <div class="dataset-meta">
- {% block resources %}
- {% block resources_outer %}
- {% if package.resources and not hide_resources %}
- <ul class="dataset-resources">
- {% block resources_inner %}
- {% for resource in h.dict_list_reduce(package.resources, 'format') %}
- <li>
- <a href="{{ h.url_for(controller='package', action='read', id=package.name) }}" class="label dataformat-label"
- data-format="{{ resource.lower() }}">{{
- resource }}</a>
- </li>
- {% endfor %}
- {% endblock %}
- </ul>
- {% endif %}
- <div class="dataset-stars">
- {% if stars>-1%}
- {% snippet "qa/stars.html", stars=stars %}
- {% endif %}
- </div>
-
- <div class="dataset-info issued">
- <p>{{ _('issued') }}: {{issued}} </p>
- </div>
- {% endblock %}
- {% endblock %}
- </div>
-
{% endblock content %}
</div>
{% endblock package_item %}
diff --git a/ckanext/odsh/templates/user/login.html b/ckanext/odsh/templates/user/login.html
index 874fd1bf692c79658d66e7946c0a69d8eb6f4e3e..d9fb0ad86d0f83803f17eba3fc3175d1825e733d 100644
--- a/ckanext/odsh/templates/user/login.html
+++ b/ckanext/odsh/templates/user/login.html
@@ -1,7 +1,16 @@
{% extends "page.html" %}
+{% set commit_hash = h.tpsh_git_commit_hash() %}
+
{% block subtitle %} {{ _('Login') }} {% endblock %}
+{% block skip %}
+<p hidden>
+ version={{ commit_hash }}
+</p>
+{{ super() }}
+{% endblock skip %}
+
{% block breadcrumb_content %}
<li class="active">{{ h.nav_link(_('Login'), controller='user', action='login') }}</li>
{% endblock %}
@@ -30,4 +39,4 @@
{% endblock %}
</section>
{% endblock %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/ckanext/odsh/tools.py b/ckanext/odsh/tools.py
index ab7c1ba60885d1a4589b1bc533f9d22e4d74d27b..d6fa856871a78fda448cd49f289be6fc024cf2c9 100644
--- a/ckanext/odsh/tools.py
+++ b/ckanext/odsh/tools.py
@@ -1,5 +1,5 @@
import os
-from ckanext.odsh.pdf_to_thumbnail.thumbnail import get_filepath_to_resource
+from ckanext.odsh.pdf_to_thumbnail.thumbnail import get_resource_path
from ckanext.odsh.lib.uploader import calculate_hash
import ckan.plugins.toolkit as toolkit
import magic
@@ -12,7 +12,7 @@ def add_attributes_resources(context, resource):
i = 0
for item in resources:
if item.get('id') == resource.get('id'):
- path = get_filepath_to_resource(resource)
+ path = get_resource_path(resource)
if os.path.exists(path):
with open(path, 'rb') as file:
diff --git a/ckanext/odsh/validation.py b/ckanext/odsh/validation.py
index a1e5338ec9e15795c7cab51ccf728c4fc38e0246..b57933968fc19251a12c727347a6aaf5e570655c 100644
--- a/ckanext/odsh/validation.py
+++ b/ckanext/odsh/validation.py
@@ -6,14 +6,13 @@ import urllib2
import json
from itertools import count
from dateutil.parser import parse
+from pylons import config
import ckan.plugins.toolkit as toolkit
import ckan.model as model
from ckan.lib.navl.dictization_functions import Missing
-from pylons import config
-
-import pdb
+from ckanext.odsh.helpers_tpsh import get_package_dict
_ = toolkit._
@@ -191,7 +190,6 @@ def known_spatial_uri(key, data, errors, context):
poly = None
# some harvesters might import a polygon directly...
- # pdb.set_trace()
poly = _extract_value(data, 'spatial')
has_old_uri = False
@@ -243,10 +241,13 @@ def known_spatial_uri(key, data, errors, context):
def _copy_spatial_uri_temp_to_extras(data):
'''
- copy the field spatial_uri_temp originating
+ copy the field spatial_uri_temp or
+ spatial_url_temp originating
from the user interface to extras
'''
spatial_uri = data.get(('__extras',)).get('spatial_uri_temp')
+ if spatial_uri is None:
+ spatial_uri = data.get(('__extras',)).get('spatial_url_temp')
is_spatial_uri_in_extras = _extract_value(data, 'spatial_uri') is not None
if not is_spatial_uri_in_extras:
next_index = next_extra_index(data)
@@ -346,6 +347,12 @@ def validate_subject(key, flattened_data, errors, context):
raise toolkit.Invalid(_('Subject must not be empty.'))
flattened_data = _convert_subjectID_to_subjectText(subject_id, flattened_data)
+def validate_relatedPackage(data):
+ if data:
+ try:
+ get_package_dict(data)
+ except logic.NotFound:
+ raise toolkit.Invalid("relatedPackage: package '{}' not found".format(data))
def get_validators():
return {
@@ -354,4 +361,5 @@ def get_validators():
'odsh_validate_extras': validate_extras,
'validate_licenseAttributionByText': validate_licenseAttributionByText,
'tpsh_validate_subject': validate_subject,
+ 'tpsh_validate_relatedPackage': validate_relatedPackage,
}
diff --git a/uls.conf b/uls.conf
deleted file mode 100644
index a94f2d66b6912e16f9b3121114f0b5e32a59e260..0000000000000000000000000000000000000000
--- a/uls.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-inputfile = /var/log/ckan/bulk.log
-inputfile = /var/log/ckan/std/gather_consumer.log
-inputfile = /var/log/ckan/std/fetch_consumer.log
-
-#outputfile = ./logs/out.uls
-
-iformat0 = %F %T,%c%c%c ERROR [%q] got exception ...%W
-nreadformat0 =... : %W
-
-# 'got exception...' ist die erste Zeile eines Tracebacks
-reg0 = got exception ...
-# alle forgenden Zeilen haben diesen Präfix
-nreadreg0 =^... :
-
-write0 = V;%F %T;%h;"odsh";"Exception";"traceback";"in %p
-nreadwrite0 = %W
\ No newline at end of file