Newer
Older
from dateutil.parser import parse

anonymous
committed
import logging
from multiline_formatter.formatter import MultilineMessagesFormatter
import os
from routes.mapper import SubMapper

Thorge Petersen
committed
from collections import OrderedDict
# imports from ckan
import ckan.lib.helpers as helpers
from ckan.lib.plugins import DefaultTranslation, DefaultDatasetForm
from ckan.logic.validators import tag_string_convert
import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.model as model
# imports from this extension
import ckanext.odsh.helpers as odsh_helpers
import ckanext.odsh.helpers_tpsh as helpers_tpsh
import ckanext.odsh.helper_pkg_dict as helper_pkg_dict
import ckanext.odsh.logic.action as action
import ckanext.odsh.validation as validation
import ckanext.odsh.search as search
from ckanext.odsh.odsh_logger import OdshLogger
import ckanext.odsh.tools as tools
from ckanext.odsh.views import default
from ckanext.odsh.views import package
from ckanext.odsh.views import user
from ckanext.odsh.views import dashboard
from ckanext.odsh.views import harvest
from ckanext.dcat import blueprints as dcat_view
class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.IDatasetForm)
plugins.implements(plugins.IFacets)
plugins.implements(plugins.IPackageController, inherit=True)
plugins.implements(plugins.IRoutes, inherit=True)
plugins.implements(plugins.ITemplateHelpers)
plugins.implements(plugins.ITranslation)
plugins.implements(plugins.IValidators)
plugins.implements(plugins.IResourceController, inherit=True)
plugins.implements(plugins.IBlueprint)
# IBlueprint
def get_blueprint(self):
log.info("OdshPlugin::get_blueprint")
# Default
bp_default = default.blueprint
rules = [
('/info_page', 'info_page', default.info_page),
('/home', 'start', default.start),
('/not_found', 'not_found', default.not_found), ]
for rule in rules:
bp_default.add_url_rule(*rule)
# DCAT
# if toolkit.asbool(toolkit.config.get('ckanext.dcat.enable_rdf_endpoints', True)):
# odsh_helpers.odsh_remove_route(map, 'dcat_catalog')
# bp_default.add_url_rule('/catalog.<any("xml", "rdf", "n3", "ttl", "jsonld"):_format>', view_func=dcat_view.read_catalog, defaults={'_format': 'xml'}, methods=['GET'])
# Package
bp_package = package.blueprint
rules = [
('/dataset/<id>/resource/<resource_id>', 'redirect_dataset_resource', package.redirect_dataset_resource), ]
for rule in rules:
bp_package.add_url_rule(*rule)
# User
bp_user = user.blueprint
bp_user.add_url_rule(u'/user', endpoint='user_index',
view_func=user.index, strict_slashes=False)
# bp_user.add_url_rule(u'/user/register', view_func=user.register)
bp_user.add_url_rule(u'/user/activity/<id>', view_func=user.activity)
bp_user.add_url_rule(u'/user/activity/<id>/<int:offset>', view_func=user.activity)
# Dashboard
bp_dashboard = dashboard.blueprint
bp_dashboard.add_url_rule(
u'/dashboard', view_func=dashboard.dashboard, strict_slashes=False, defaults={
u'offset': 0
})
bp_dashboard.add_url_rule(
u'/dashboard/<int:offset>', view_func=dashboard.dashboard)
bp_dashboard.add_url_rule(
u'/dashboard/datasets', view_func=dashboard.dashboard_datasets)
#
bp_harvest = harvest.blueprint
bp_harvest.add_url_rule(u'/harvest/new', view_func=dashboard.new)
return [bp_default, bp_package, bp_user, bp_dashboard, bp_harvest]
def get_actions(self):
return {'package_create': action.odsh_package_create,
'user_update': action.tpsh_user_update,
'resource_create': action.odsh_resource_create, }
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'ckanext-odsh')
def package_types(self):
# This plugin doesn't handle any special package types, it just
# registers itself as the default (above).
return []
def is_fallback(self):
# Return True to register this plugin as the default handler for
# package types not handled by any other IDatasetForm plugin.
return True
def create_package_schema(self):
schema = super(OdshPlugin, self).create_package_schema()
self._update_schema(schema)
self._tpsh_update_create_or_update_package_schema(schema)
return schema
def update_package_schema(self):
schema = super(OdshPlugin, self).update_package_schema()
self._update_schema(schema)
self._tpsh_update_create_or_update_package_schema(schema)
return schema
def show_package_schema(self):
schema = super(OdshPlugin, self).show_package_schema()
self._tpsh_update_show_package_schema(schema)
return schema

anonymous
committed
def _update_schema(self, schema):
for field in ['title', 'license_id']:
schema.update({field: [toolkit.get_converter('not_empty')]})

Thorge Petersen
committed
schema.update({
'reference': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
})
for i, item in enumerate(schema['tags']['name']):
if item == toolkit.get_validator('tag_name_validator'):
schema['tags']['name'][i] = toolkit.get_validator(
'odsh_tag_name_validator')
for i, item in enumerate(schema['tag_string']):
if item == tag_string_convert:
schema['tag_string'][i] = validation.tag_string_convert

anonymous
committed
schema['resources'].update({
'url': [toolkit.get_converter('not_empty')],
'format': [toolkit.get_converter('odsh_validate_format')],

anonymous
committed
schema['extras'].update({
'key': [
toolkit.get_converter('known_spatial_uri'),
toolkit.get_converter('validate_licenseAttributionByText'),
]
})
schema.update(
{'__extras': [toolkit.get_converter('odsh_validate_extras')]})

anonymous
committed
def _tpsh_update_create_or_update_package_schema(self, schema):
schema.update({
'language': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'thumbnail': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
],
'relatedPackage': [
toolkit.get_validator('tpsh_validate_relatedPackage'),
toolkit.get_converter('convert_to_extras')
],
'accessibility': [
toolkit.get_validator('ignore_missing'),
toolkit.get_converter('convert_to_extras')
]
def _tpsh_update_show_package_schema(self, schema):
schema.update({
'language': [
toolkit.get_converter('convert_from_extras'),
toolkit.get_validator('ignore_missing')
],
'thumbnail': [
toolkit.get_converter('convert_from_extras'),
toolkit.get_validator('ignore_missing')
],
'relatedPackage': [
toolkit.get_converter('convert_from_extras'),
toolkit.get_validator('ignore_missing')
],
'accessibility': [
toolkit.get_converter('convert_from_extras'),
toolkit.get_validator('ignore_missing')
],
return OrderedDict({'organization': _('Organizations'),
'groups': _('Category'),
'res_format': _('File format'),
'license_title': _('License'),
def group_facets(self, facets_dict, group_type, package_type):
return OrderedDict({'organization': _('Organizations'),
'res_format': _('File format'),
'license_title': _('License'),
'groups': _('Category')})
def organization_facets(self, facets_dict, organization_type, package_type):
return OrderedDict({'organization': _('Organizations'),
'res_format': _('File format'),
'license_title': _('License'),
'groups': _('Category')})
def after_show(self, context, pkg_dict):
'''
corrects missing relationships in pkg dict
adds the following key-value-pairs to pkg_dict:
# key: 'is_new', value: True if the dataset has been created within the last month
'''
pkg_dict = helpers_tpsh.correct_missing_relationship(
pkg_dict,
helpers_tpsh.get_pkg_relationships_from_model(pkg_dict)
)
self._update_is_new_in_pkg_dict(pkg_dict)
def before_view(self, pkg_dict):
'''
adds the following key-value-pairs to pkg_dict:
# key: 'is_new', value: True if the dataset has been created within the last month
'''
self._update_is_new_in_pkg_dict(pkg_dict)
return pkg_dict
def after_create(self, context, resource):
if resource.get('package_id'):
tools.add_attributes_resources(context, resource)
def after_update(self, context, resource):
if resource.get('package_id'):
tools.add_attributes_resources(context, resource)
@staticmethod
def _update_is_new_in_pkg_dict(pkg_dict):
is_new = HelperPgkDict(pkg_dict).is_package_new()
pkg_dict.update({'is_new': is_new})
def before_index(self, dict_pkg):
# make special date fields solr conform
fields = ["issued", "temporal_start", "temporal_end"]
for field in fields:
field = 'extras_' + field
if field in dict_pkg and dict_pkg[field]:
d = parse(dict_pkg[field])
dict_pkg[field] = '{0.year:04d}-{0.month:02d}-{0.day:02d}T00:00:00Z'.format(
d)
map.redirect('/dataset/{id}/resource/{resource_id}', '/dataset/{id}')
with SubMapper(map, controller='ckanext.odsh.controller:OdshFeedController') as m:
m.connect('/feeds/custom.atom', action='custom')
# redirect all user routes to custom controller
# with SubMapper(map, controller='ckanext.odsh.controller:OdshUserController') as m:
# m.connect('/user/edit', action='edit')
# m.connect(
# 'user_edit', '/user/edit/{id:.*}', action='edit', ckan_icon='cog')
# m.connect('user_delete', '/user/delete/{id}', action='delete')
# m.connect('/user/reset/{id:.*}', action='perform_reset')
# m.connect('/user/reset', action='request_reset')
# m.connect('register', '/user/register', action='register')
# m.connect('login', '/user/login', action='login')
# m.connect('/user/_logout', action='logout')
# m.connect('/user/logged_in', action='logged_in')
# m.connect('/user/logged_out', action='logged_out')
# m.connect('/user/logged_out_redirect', action='logged_out_page')
# m.connect('user_datasets', '/user/{id:(?!(generate_key|activity)).*}', action='read',
# ckan_icon='sitemap')
'/datarequest/new',
controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI',
action='new',
conditions=dict(method=['GET', 'POST']),
'/datarequest/{id}',
controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI',
action='comment',
conditions=dict(method=['GET', 'POST']),
def get_helpers(self):
# Template helper function names should begin with the name of the
# extension they belong to, to avoid clashing with functions from
# other extensions.
return {
'odsh_main_groups': odsh_helpers.odsh_main_groups,
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
'odsh_now': odsh_helpers.odsh_now,
'odsh_group_id_selected': odsh_helpers.odsh_group_id_selected,
'odsh_get_facet_items_dict': odsh_helpers.odsh_get_facet_items_dict,
'odsh_openness_score_dataset_html': odsh_helpers.odsh_openness_score_dataset_html,
'odsh_get_resource_details': odsh_helpers.odsh_get_resource_details,
'odsh_get_resource_views': odsh_helpers.odsh_get_resource_views,
'odsh_get_bounding_box': odsh_helpers.odsh_get_bounding_box,
'odsh_get_spatial_text': odsh_helpers.odsh_get_spatial_text,
'odsh_render_datetime': odsh_helpers.odsh_render_datetime,
'odsh_upload_known_formats': odsh_helpers.odsh_upload_known_formats,
'odsh_encodeurl': odsh_helpers.odsh_encodeurl,
'odsh_extract_error': odsh_helpers.odsh_extract_error,
'odsh_extract_error_new': odsh_helpers.odsh_extract_error_new,
'odsh_extract_value_from_extras': odsh_helpers.odsh_extract_value_from_extras,
'odsh_create_checksum': odsh_helpers.odsh_create_checksum,
'presorted_license_options': odsh_helpers.presorted_license_options,
'odsh_has_more_facets': odsh_helpers.odsh_has_more_facets,
'odsh_public_url': odsh_helpers.odsh_public_url,
'odsh_spatial_extends_available': odsh_helpers.spatial_extends_available,
'odsh_public_resource_url': odsh_helpers.odsh_public_resource_url,
'odsh_get_version_id': odsh_helpers.odsh_get_version_id,
'odsh_show_testbanner': odsh_helpers.odsh_show_testbanner,
'odsh_is_slave': odsh_helpers.odsh_is_slave,
'tpsh_get_daterange_prettified': helper_pkg_dict.get_daterange_prettified,
'tpsh_get_language_of_package': helpers_tpsh.get_language_of_package,
'get_language_icon': helpers_tpsh.get_language_icon,
'short_name_for_category': odsh_helpers.short_name_for_category,
'get_spatial_for_selection': helpers_tpsh.get_spatial_for_selection,
'get_subject_for_selection': helpers_tpsh.get_subject_for_selection,
'get_language_for_selection': helpers_tpsh.get_language_for_selection,
'tpsh_get_resource_size': helpers_tpsh.get_resource_size,
'tpsh_get_address_org':helpers_tpsh.get_address_org,
'tpsh_get_body_mail':helpers_tpsh.get_body_mail,

Thorge Petersen
committed
'odsh_load_mdk_sample_dataset': odsh_helpers.odsh_load_mdk_sample_dataset,
'odsh_load_raw_mdk_sample_dataset': odsh_helpers.odsh_load_raw_mdk_sample_dataset,
# Add the custom parameters to Solr's facet queries
# use several daterange queries agains temporal_start and temporal_end field
# TODO: use field of type date_range in solr index instead
scores = [['0OL'], ['0OL', '1RE'], ['0OL', '1RE', '2OF'], [
'0OL', '1RE', '2OF', '3URI'], ['0OL', '1RE', '2OF', '3URI', '4LD']]
def map_qa_score(self, dict_pkg):
if 'validated_data_dict' in dict_pkg and 'openness_score' in dict_pkg['validated_data_dict']:
d = json.loads(dict_pkg['validated_data_dict'])
score = -1
for r in d['resources']:
if 'qa' in r:
if 'openness_score' in r['qa']:
s = int(r['qa']['openness_score'])
if s > score:
score = s
if score > 0:
dict_pkg['openness'] = OdshPlugin.scores[score-1]