# import from third parties from dateutil.parser import parse import json import logging from collections import OrderedDict # imports from ckan from ckan.lib.plugins import DefaultTranslation, DefaultDatasetForm from ckan.logic.validators import tag_string_convert import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit # imports from this extension import ckanext.odsh.helpers as odsh_helpers import ckanext.odsh.helpers_tpsh as helpers_tpsh import ckanext.odsh.helper_pkg_dict as helper_pkg_dict from .helper_pkg_dict import HelperPgkDict import ckanext.odsh.logic.action as action import ckanext.odsh.validation as validation import ckanext.odsh.search as search import ckanext.odsh.tools as tools from ckanext.odsh.views import default from ckanext.odsh.views import package from ckanext.odsh.views import user from ckanext.odsh.views import dashboard from ckanext.odsh.views import harvest from ckanext.odsh.views import feed from ckanext.dcat import blueprints as dcat_view log = logging.getLogger(__name__) _ = toolkit._ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm): plugins.implements(plugins.IActions) plugins.implements(plugins.IConfigurer) plugins.implements(plugins.IDatasetForm) plugins.implements(plugins.IFacets) plugins.implements(plugins.IPackageController, inherit=True) plugins.implements(plugins.ITemplateHelpers) plugins.implements(plugins.ITranslation) plugins.implements(plugins.IValidators) plugins.implements(plugins.IResourceController, inherit=True) plugins.implements(plugins.IBlueprint) # IBlueprint def get_blueprint(self): log.info("OdshPlugin::get_blueprint") # Default bp_default = default.blueprint rules = [ ('/info_page', 'info_page', default.info_page), ('/home', 'start', default.start), ('/not_found', 'not_found', default.not_found), ] for rule in rules: bp_default.add_url_rule(*rule) # DCAT # if toolkit.asbool(toolkit.config.get('ckanext.dcat.enable_rdf_endpoints', True)): # odsh_helpers.odsh_remove_route(map, 'dcat_catalog') # bp_default.add_url_rule('/catalog.<any("xml", "rdf", "n3", "ttl", "jsonld"):_format>', view_func=dcat_view.read_catalog, defaults={'_format': 'xml'}, methods=['GET']) # Package bp_package = package.blueprint rules = [ ('/dataset/<id>/resource/<resource_id>', 'redirect_dataset_resource', package.redirect_dataset_resource), ] for rule in rules: bp_package.add_url_rule(*rule) # User bp_user = user.blueprint bp_user.add_url_rule(u'/user', endpoint='user_index', view_func=user.index, strict_slashes=False) # bp_user.add_url_rule(u'/user/register', view_func=user.register) bp_user.add_url_rule(u'/user/activity/<id>', view_func=user.activity) bp_user.add_url_rule(u'/user/activity/<id>/<int:offset>', view_func=user.activity) # Dashboard bp_dashboard = dashboard.blueprint bp_dashboard.add_url_rule( u'/dashboard', view_func=dashboard.dashboard, strict_slashes=False, defaults={ u'offset': 0 }) bp_dashboard.add_url_rule( u'/dashboard/<int:offset>', view_func=dashboard.dashboard) bp_dashboard.add_url_rule( u'/dashboard/datasets', view_func=dashboard.dashboard_datasets) # Harvest bp_harvest = harvest.blueprint bp_harvest.add_url_rule(u'/harvest', view_func=harvest.search, strict_slashes=False) bp_harvest.add_url_rule(u'/harvest/<id>', view_func=harvest.read) bp_harvest.add_url_rule(u'/harvest/about/<id>', view_func=harvest.about) bp_harvest.add_url_rule(u'/harvest/new', view_func=harvest.new) # Feed bp_feed = feed.blueprint bp_feed.add_url_rule(u'/feeds/custom.atom', methods=[u'GET'], view_func=feed.custom) return [bp_default, bp_package, bp_user, bp_dashboard, bp_harvest, bp_feed] # IActions def get_actions(self): return {'package_create': action.odsh_package_create, 'user_update': action.tpsh_user_update, 'user_create': action.odsh_user_create, 'resource_create': action.odsh_resource_create, } # IConfigurer def update_config(self, config_): toolkit.add_template_directory(config_, 'templates') toolkit.add_public_directory(config_, 'public') toolkit.add_resource('fanstatic', 'ckanext-odsh') def after_map(self, map): return map def package_types(self): # This plugin doesn't handle any special package types, it just # registers itself as the default (above). return [] def is_fallback(self): # Return True to register this plugin as the default handler for # package types not handled by any other IDatasetForm plugin. return True def create_package_schema(self): schema = super(OdshPlugin, self).create_package_schema() self._update_schema(schema) self._tpsh_update_create_or_update_package_schema(schema) return schema def update_package_schema(self): schema = super(OdshPlugin, self).update_package_schema() self._update_schema(schema) self._tpsh_update_create_or_update_package_schema(schema) return schema def show_package_schema(self): schema = super(OdshPlugin, self).show_package_schema() self._tpsh_update_show_package_schema(schema) return schema def _update_schema(self, schema): for field in ['title', 'license_id']: schema.update({field: [toolkit.get_converter('not_empty')]}) schema.update({ 'reference': [ toolkit.get_validator('ignore_missing'), toolkit.get_converter('convert_to_extras') ] }) for i, item in enumerate(schema['tags']['name']): if item == toolkit.get_validator('tag_name_validator'): schema['tags']['name'][i] = toolkit.get_validator( 'odsh_tag_name_validator') for i, item in enumerate(schema['tag_string']): if item == tag_string_convert: schema['tag_string'][i] = validation.tag_string_convert schema['resources'].update({ 'url': [toolkit.get_converter('not_empty')], 'format': [toolkit.get_converter('odsh_validate_format')], }) schema['extras'].update({ 'key': [ toolkit.get_converter('known_spatial_uri'), toolkit.get_converter('validate_licenseAttributionByText'), ] }) schema.update( {'__extras': [toolkit.get_converter('odsh_validate_extras')]}) def _tpsh_update_create_or_update_package_schema(self, schema): schema.update({ 'language': [ toolkit.get_validator('ignore_missing'), toolkit.get_converter('convert_to_extras') ], 'thumbnail': [ toolkit.get_validator('ignore_missing'), toolkit.get_converter('convert_to_extras') ], 'relatedPackage': [ toolkit.get_validator('tpsh_validate_relatedPackage'), toolkit.get_converter('convert_to_extras') ], 'accessibility': [ toolkit.get_validator('ignore_missing'), toolkit.get_converter('convert_to_extras') ] }) return schema def _tpsh_update_show_package_schema(self, schema): schema.update({ 'language': [ toolkit.get_converter('convert_from_extras'), toolkit.get_validator('ignore_missing') ], 'thumbnail': [ toolkit.get_converter('convert_from_extras'), toolkit.get_validator('ignore_missing') ], 'relatedPackage': [ toolkit.get_converter('convert_from_extras'), toolkit.get_validator('ignore_missing') ], 'accessibility': [ toolkit.get_converter('convert_from_extras'), toolkit.get_validator('ignore_missing') ], }) return schema # IFacets def dataset_facets(self, facets_dict, package_type): return OrderedDict({'organization': _('Organizations'), 'groups': _('Category'), 'res_format': _('File format'), 'license_title': _('License'), # 'tags': _('Tags'), 'openness': _('Open-Data-Eigenschaften') }) def group_facets(self, facets_dict, group_type, package_type): return OrderedDict({'organization': _('Organizations'), 'res_format': _('File format'), 'license_title': _('License'), 'groups': _('Category')}) def organization_facets(self, facets_dict, organization_type, package_type): return OrderedDict({'organization': _('Organizations'), 'res_format': _('File format'), 'license_title': _('License'), 'groups': _('Category')}) # IPackageController def after_show(self, context, pkg_dict): ''' corrects missing relationships in pkg dict adds the following key-value-pairs to pkg_dict: # key: 'is_new', value: True if the dataset has been created within the last month ''' pkg_dict = helpers_tpsh.correct_missing_relationship( pkg_dict, helpers_tpsh.get_pkg_relationships_from_model(pkg_dict) ) self._update_is_new_in_pkg_dict(pkg_dict) return pkg_dict def before_view(self, pkg_dict): ''' adds the following key-value-pairs to pkg_dict: # key: 'is_new', value: True if the dataset has been created within the last month ''' self._update_is_new_in_pkg_dict(pkg_dict) return pkg_dict def after_create(self, context, resource): if resource.get('package_id'): tools.add_attributes_resources(context, resource) def after_update(self, context, resource): if resource.get('package_id'): tools.add_attributes_resources(context, resource) @staticmethod def _update_is_new_in_pkg_dict(pkg_dict): is_new = HelperPgkDict(pkg_dict).is_package_new() pkg_dict.update({'is_new': is_new}) def before_index(self, dict_pkg): # make special date fields solr conform fields = ["issued", "temporal_start", "temporal_end"] for field in fields: field = 'extras_' + field if field in dict_pkg and dict_pkg[field]: d = parse(dict_pkg[field]) dict_pkg[field] = '{0.year:04d}-{0.month:02d}-{0.day:02d}T00:00:00Z'.format( d) self.map_qa_score(dict_pkg) return dict_pkg # ITemplateHelpers def get_helpers(self): # Template helper function names should begin with the name of the # extension they belong to, to avoid clashing with functions from # other extensions. return { 'odsh_main_groups': odsh_helpers.odsh_main_groups, 'odsh_now': odsh_helpers.odsh_now, 'odsh_group_id_selected': odsh_helpers.odsh_group_id_selected, 'odsh_get_facet_items_dict': odsh_helpers.odsh_get_facet_items_dict, 'odsh_openness_score_dataset_html': odsh_helpers.odsh_openness_score_dataset_html, 'odsh_get_resource_details': odsh_helpers.odsh_get_resource_details, 'odsh_get_resource_views': odsh_helpers.odsh_get_resource_views, 'odsh_get_bounding_box': odsh_helpers.odsh_get_bounding_box, 'odsh_get_spatial_text': odsh_helpers.odsh_get_spatial_text, 'odsh_render_datetime': odsh_helpers.odsh_render_datetime, 'odsh_upload_known_formats': odsh_helpers.odsh_upload_known_formats, 'odsh_encodeurl': odsh_helpers.odsh_encodeurl, 'odsh_extract_error': odsh_helpers.odsh_extract_error, 'odsh_extract_error_new': odsh_helpers.odsh_extract_error_new, 'odsh_extract_value_from_extras': odsh_helpers.odsh_extract_value_from_extras, 'odsh_create_checksum': odsh_helpers.odsh_create_checksum, 'presorted_license_options': odsh_helpers.presorted_license_options, 'odsh_has_more_facets': odsh_helpers.odsh_has_more_facets, 'odsh_public_url': odsh_helpers.odsh_public_url, 'odsh_spatial_extends_available': odsh_helpers.spatial_extends_available, 'odsh_public_resource_url': odsh_helpers.odsh_public_resource_url, 'odsh_get_version_id': odsh_helpers.odsh_get_version_id, 'odsh_show_testbanner': odsh_helpers.odsh_show_testbanner, 'odsh_is_slave': odsh_helpers.odsh_is_slave, 'tpsh_get_daterange_prettified': helper_pkg_dict.get_daterange_prettified, 'tpsh_get_language_of_package': helpers_tpsh.get_language_of_package, 'get_language_icon': helpers_tpsh.get_language_icon, 'short_name_for_category': odsh_helpers.short_name_for_category, 'get_spatial_for_selection': helpers_tpsh.get_spatial_for_selection, 'get_subject_for_selection': helpers_tpsh.get_subject_for_selection, 'get_language_for_selection': helpers_tpsh.get_language_for_selection, 'tpsh_get_resource_size': helpers_tpsh.get_resource_size, 'tpsh_get_address_org':helpers_tpsh.get_address_org, 'tpsh_get_body_mail':helpers_tpsh.get_body_mail, 'odsh_load_mdk_sample_dataset': odsh_helpers.odsh_load_mdk_sample_dataset, 'odsh_load_raw_mdk_sample_dataset': odsh_helpers.odsh_load_raw_mdk_sample_dataset, } # IValidators def get_validators(self): return validation.get_validators() # Add the custom parameters to Solr's facet queries # use several daterange queries agains temporal_start and temporal_end field # TODO: use field of type date_range in solr index instead def before_search(self, search_params): return search.before_search(search_params) scores = [['0OL'], ['0OL', '1RE'], ['0OL', '1RE', '2OF'], [ '0OL', '1RE', '2OF', '3URI'], ['0OL', '1RE', '2OF', '3URI', '4LD']] def map_qa_score(self, dict_pkg): if 'validated_data_dict' in dict_pkg and 'openness_score' in dict_pkg['validated_data_dict']: d = json.loads(dict_pkg['validated_data_dict']) score = -1 for r in d['resources']: if 'qa' in r: if 'openness_score' in r['qa']: s = int(r['qa']['openness_score']) if s > score: score = s if score > 0: dict_pkg['openness'] = OdshPlugin.scores[score-1]