Skip to content
Snippets Groups Projects
plugin.py 17 KiB
Newer Older
  • Learn to ignore specific revisions
  • root's avatar
    root committed
    # import from third parties
    
    from dateutil.parser import parse
    
    root's avatar
    root committed
    import json
    
    root's avatar
    root committed
    from multiline_formatter.formatter import MultilineMessagesFormatter
    import os
    from routes.mapper import SubMapper
    
    anonymous's avatar
    anonymous committed
    import sys
    
    from flask import Blueprint
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
    # imports from ckan
    import ckan.lib.helpers as helpers
    from ckan.lib.plugins import DefaultTranslation, DefaultDatasetForm
    from ckan.logic.validators import tag_string_convert
    import ckan.plugins as plugins
    import ckan.plugins.toolkit as toolkit
    import ckan.model as model
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
    # imports from this extension
    import ckanext.odsh.helpers as odsh_helpers
    import ckanext.odsh.helpers_tpsh as helpers_tpsh
    import ckanext.odsh.helper_pkg_dict as helper_pkg_dict
    
    Thorge Petersen's avatar
    Thorge Petersen committed
    from .helper_pkg_dict import HelperPgkDict
    
    root's avatar
    root committed
    import ckanext.odsh.logic.action as action
    import ckanext.odsh.validation as validation
    import ckanext.odsh.search as search
    from ckanext.odsh.odsh_logger import OdshLogger
    import ckanext.odsh.tools as tools
    
    from ckanext.odsh.views import default
    from ckanext.odsh.views import package
    from ckanext.odsh.views import user
    from ckanext.odsh.views import dashboard
    
    from ckanext.odsh.views import harvest
    
    from ckanext.dcat import blueprints as dcat_view
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
    log = logging.getLogger(__name__)
    
    anonymous's avatar
    anonymous committed
    
    
    anonymous's avatar
    anonymous committed
    _ = toolkit._
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
    class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm):
    
    anonymous's avatar
    anonymous committed
        plugins.implements(plugins.IActions)
    
    root's avatar
    root committed
        plugins.implements(plugins.IConfigurer)
        plugins.implements(plugins.IDatasetForm)
        plugins.implements(plugins.IFacets)
        plugins.implements(plugins.IPackageController, inherit=True)
        plugins.implements(plugins.IRoutes, inherit=True)
        plugins.implements(plugins.ITemplateHelpers)
        plugins.implements(plugins.ITranslation)
        plugins.implements(plugins.IValidators)
        plugins.implements(plugins.IResourceController, inherit=True)
    
        plugins.implements(plugins.IBlueprint)
    
        # IBlueprint
        def get_blueprint(self):
            log.info("OdshPlugin::get_blueprint")
    
            # Default
            bp_default = default.blueprint
            rules = [
                ('/info_page', 'info_page', default.info_page),
                ('/home', 'start', default.start),
                ('/not_found', 'not_found', default.not_found), ]
            for rule in rules:
                bp_default.add_url_rule(*rule)
    
            # DCAT
            # if toolkit.asbool(toolkit.config.get('ckanext.dcat.enable_rdf_endpoints', True)):
            #     odsh_helpers.odsh_remove_route(map, 'dcat_catalog')
            #     bp_default.add_url_rule('/catalog.<any("xml", "rdf", "n3", "ttl", "jsonld"):_format>', view_func=dcat_view.read_catalog, defaults={'_format': 'xml'}, methods=['GET'])
    
            # Package
            bp_package = package.blueprint
            rules = [
                ('/dataset/<id>/resource/<resource_id>', 'redirect_dataset_resource', package.redirect_dataset_resource), ]
            for rule in rules:
                bp_package.add_url_rule(*rule)
    
            # User
            bp_user = user.blueprint
            bp_user.add_url_rule(u'/user', endpoint='user_index',
                                 view_func=user.index, strict_slashes=False)
    
    Thorge Petersen's avatar
    Thorge Petersen committed
            # bp_user.add_url_rule(u'/user/register', view_func=user.register)
    
            bp_user.add_url_rule(u'/user/activity/<id>', view_func=user.activity)
            bp_user.add_url_rule(u'/user/activity/<id>/<int:offset>', view_func=user.activity)
            
            # Dashboard
            bp_dashboard = dashboard.blueprint
            bp_dashboard.add_url_rule(
                u'/dashboard', view_func=dashboard.dashboard, strict_slashes=False, defaults={
                    u'offset': 0
                })
            bp_dashboard.add_url_rule(
                u'/dashboard/<int:offset>', view_func=dashboard.dashboard)
            bp_dashboard.add_url_rule(
                u'/dashboard/datasets', view_func=dashboard.dashboard_datasets)
    
    
            #
            bp_harvest = harvest.blueprint
            bp_harvest.add_url_rule(u'/harvest/new', view_func=dashboard.new)
            return [bp_default, bp_package, bp_user, bp_dashboard, bp_harvest]
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        # IActions
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        def get_actions(self):
            return {'package_create': action.odsh_package_create,
    
                    'user_update': action.tpsh_user_update,
    
                    'user_create': action.odsh_user_create,
    
                    'resource_create': action.odsh_resource_create, }
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        # IConfigurer
    
    anonymous's avatar
    anonymous committed
    
        def update_config(self, config_):
    
    root's avatar
    root committed
            toolkit.add_template_directory(config_, 'templates')
            toolkit.add_public_directory(config_, 'public')
    
            toolkit.add_resource('fanstatic', 'ckanext-odsh')
    
    root's avatar
    root committed
        # IDatasetForm
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        def package_types(self):
            # This plugin doesn't handle any special package types, it just
            # registers itself as the default (above).
            return []
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        def is_fallback(self):
            # Return True to register this plugin as the default handler for
            # package types not handled by any other IDatasetForm plugin.
            return True
    
    root's avatar
    root committed
        def create_package_schema(self):
            schema = super(OdshPlugin, self).create_package_schema()
            self._update_schema(schema)
            self._tpsh_update_create_or_update_package_schema(schema)
            return schema
    
    root's avatar
    root committed
        def update_package_schema(self):
            schema = super(OdshPlugin, self).update_package_schema()
            self._update_schema(schema)
            self._tpsh_update_create_or_update_package_schema(schema)
            return schema
    
    root's avatar
    root committed
        def show_package_schema(self):
            schema = super(OdshPlugin, self).show_package_schema()
            self._tpsh_update_show_package_schema(schema)
            return schema
    
    root's avatar
    root committed
        def _update_schema(self, schema):
            for field in ['title', 'license_id']:
                schema.update({field: [toolkit.get_converter('not_empty')]})
    
            schema.update({
                'reference': [
                    toolkit.get_validator('ignore_missing'),
                    toolkit.get_converter('convert_to_extras')
                ]
            })
    
    root's avatar
    root committed
            for i, item in enumerate(schema['tags']['name']):
                if item == toolkit.get_validator('tag_name_validator'):
                    schema['tags']['name'][i] = toolkit.get_validator(
                        'odsh_tag_name_validator')
            for i, item in enumerate(schema['tag_string']):
                if item == tag_string_convert:
                    schema['tag_string'][i] = validation.tag_string_convert
    
    root's avatar
    root committed
            schema['resources'].update({
                'url': [toolkit.get_converter('not_empty')],
    
                'format': [toolkit.get_converter('odsh_validate_format')],
    
    root's avatar
    root committed
            })
    
    root's avatar
    root committed
            schema['extras'].update({
                'key': [
                    toolkit.get_converter('known_spatial_uri'),
                    toolkit.get_converter('validate_licenseAttributionByText'),
                ]
            })
            schema.update(
                {'__extras':  [toolkit.get_converter('odsh_validate_extras')]})
    
    root's avatar
    root committed
        def _tpsh_update_create_or_update_package_schema(self, schema):
            schema.update({
                'language': [
                    toolkit.get_validator('ignore_missing'),
                    toolkit.get_converter('convert_to_extras')
                ],
                'thumbnail': [
                    toolkit.get_validator('ignore_missing'),
                    toolkit.get_converter('convert_to_extras')
                ],
    
                'relatedPackage': [
                    toolkit.get_validator('tpsh_validate_relatedPackage'),
                    toolkit.get_converter('convert_to_extras')
                ],
                'accessibility': [
                    toolkit.get_validator('ignore_missing'),
                    toolkit.get_converter('convert_to_extras')
                ]
    
    root's avatar
    root committed
            })
            return schema
    
    root's avatar
    root committed
        def _tpsh_update_show_package_schema(self, schema):
            schema.update({
                'language': [
                    toolkit.get_converter('convert_from_extras'),
                    toolkit.get_validator('ignore_missing')
                ],
                'thumbnail': [
                    toolkit.get_converter('convert_from_extras'),
                    toolkit.get_validator('ignore_missing')
                ],
    
                'relatedPackage': [
                    toolkit.get_converter('convert_from_extras'),
                    toolkit.get_validator('ignore_missing')
                ],
                'accessibility': [
                    toolkit.get_converter('convert_from_extras'),
                    toolkit.get_validator('ignore_missing')
                ],
    
    root's avatar
    root committed
            })
            return schema
    
    root's avatar
    root committed
        # IFacets
    
    root's avatar
    root committed
        def dataset_facets(self, facets_dict, package_type):
    
    Thorge Petersen's avatar
    Thorge Petersen committed
            return OrderedDict({'organization': _('Organizations'),
                                'groups': _('Category'),
                                'res_format': _('File format'),
                                'license_title': _('License'),
    
                                #                            'tags': _('Tags'),
    
    root's avatar
    root committed
                                'openness': _('Open-Data-Eigenschaften')
                                })
    
    root's avatar
    root committed
        def group_facets(self, facets_dict, group_type, package_type):
    
    Thorge Petersen's avatar
    Thorge Petersen committed
            return OrderedDict({'organization': _('Organizations'),
                                'res_format': _('File format'),
                                'license_title': _('License'),
                                'groups': _('Category')})
    
    root's avatar
    root committed
        def organization_facets(self, facets_dict, organization_type, package_type):
    
    Thorge Petersen's avatar
    Thorge Petersen committed
            return OrderedDict({'organization': _('Organizations'),
                                'res_format': _('File format'),
                                'license_title': _('License'),
                                'groups': _('Category')})
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        # IPackageController
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        def after_show(self, context, pkg_dict):
            '''
            corrects missing relationships in pkg dict
            adds the following key-value-pairs to pkg_dict:
            # key: 'is_new', value: True if the dataset has been created within the last month
            '''
            pkg_dict = helpers_tpsh.correct_missing_relationship(
                pkg_dict,
                helpers_tpsh.get_pkg_relationships_from_model(pkg_dict)
            )
            self._update_is_new_in_pkg_dict(pkg_dict)
    
    root's avatar
    root committed
            return pkg_dict
    
    root's avatar
    root committed
        def before_view(self, pkg_dict):
            '''
            adds the following key-value-pairs to pkg_dict:
            # key: 'is_new', value: True if the dataset has been created within the last month
            '''
            self._update_is_new_in_pkg_dict(pkg_dict)
            return pkg_dict
    
    root's avatar
    root committed
        def after_create(self, context, resource):
            if resource.get('package_id'):
                tools.add_attributes_resources(context, resource)
    
    root's avatar
    root committed
        def after_update(self, context, resource):
            if resource.get('package_id'):
                tools.add_attributes_resources(context, resource)
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        @staticmethod
        def _update_is_new_in_pkg_dict(pkg_dict):
            is_new = HelperPgkDict(pkg_dict).is_package_new()
            pkg_dict.update({'is_new': is_new})
    
        def before_index(self, dict_pkg):
            # make special date fields solr conform
            fields = ["issued", "temporal_start", "temporal_end"]
            for field in fields:
                field = 'extras_' + field
                if field in dict_pkg and dict_pkg[field]:
                    d = parse(dict_pkg[field])
                    dict_pkg[field] = '{0.year:04d}-{0.month:02d}-{0.day:02d}T00:00:00Z'.format(
                        d)
    
    root's avatar
    root committed
            self.map_qa_score(dict_pkg)
    
    root's avatar
    root committed
            return dict_pkg
    
    anonymous's avatar
    anonymous committed
    
    
    root's avatar
    root committed
        # IRoutes
    
    
            map.redirect('/dataset/{id}/resource/{resource_id}', '/dataset/{id}')
    
    
            with SubMapper(map, controller='ckanext.odsh.controller:OdshFeedController') as m:
                m.connect('/feeds/custom.atom', action='custom')
    
    anonymous's avatar
    anonymous committed
    
    
            # redirect all user routes to custom controller
    
            # with SubMapper(map, controller='ckanext.odsh.controller:OdshUserController') as m:
            #     m.connect('/user/edit', action='edit')
            #     m.connect(
            #         'user_edit', '/user/edit/{id:.*}', action='edit', ckan_icon='cog')
            #     m.connect('user_delete', '/user/delete/{id}', action='delete')
            #     m.connect('/user/reset/{id:.*}', action='perform_reset')
            #     m.connect('/user/reset', action='request_reset')
            #     m.connect('register', '/user/register', action='register')
            #     m.connect('login', '/user/login', action='login')
            #     m.connect('/user/_logout', action='logout')
            #     m.connect('/user/logged_in', action='logged_in')
            #     m.connect('/user/logged_out', action='logged_out')
            #     m.connect('/user/logged_out_redirect', action='logged_out_page')
            #     m.connect('user_datasets', '/user/{id:(?!(generate_key|activity)).*}', action='read',
            #               ckan_icon='sitemap')
    
    root's avatar
    root committed
            map.connect(
    
                'comment_datarequest',
    
    root's avatar
    root committed
                '/datarequest/new',
                controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI',
    
                action='new',
                conditions=dict(method=['GET', 'POST']),
    
    root's avatar
    root committed
                ckan_icon='comment'
            )
            map.connect(
    
                'comment_datarequest',
    
    root's avatar
    root committed
                '/datarequest/{id}',
                controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI',
    
                action='comment',
                conditions=dict(method=['GET', 'POST']),
    
    root's avatar
    root committed
                ckan_icon='comment'
            )
            return map
    
    root's avatar
    root committed
        def after_map(self, map):
    
    root's avatar
    root committed
        # ITemplateHelpers
    
    root's avatar
    root committed
        def get_helpers(self):
            # Template helper function names should begin with the name of the
            # extension they belong to, to avoid clashing with functions from
            # other extensions.
    
            return {
    	        'odsh_main_groups': odsh_helpers.odsh_main_groups,
    
    root's avatar
    root committed
                    'odsh_now': odsh_helpers.odsh_now,
                    'odsh_group_id_selected': odsh_helpers.odsh_group_id_selected,
                    'odsh_get_facet_items_dict': odsh_helpers.odsh_get_facet_items_dict,
                    'odsh_openness_score_dataset_html': odsh_helpers.odsh_openness_score_dataset_html,
                    'odsh_get_resource_details': odsh_helpers.odsh_get_resource_details,
                    'odsh_get_resource_views': odsh_helpers.odsh_get_resource_views,
                    'odsh_get_bounding_box': odsh_helpers.odsh_get_bounding_box,
                    'odsh_get_spatial_text': odsh_helpers.odsh_get_spatial_text,
                    'odsh_render_datetime': odsh_helpers.odsh_render_datetime,
                    'odsh_upload_known_formats': odsh_helpers.odsh_upload_known_formats,
                    'odsh_encodeurl': odsh_helpers.odsh_encodeurl,
                    'odsh_extract_error': odsh_helpers.odsh_extract_error,
                    'odsh_extract_error_new': odsh_helpers.odsh_extract_error_new,
                    'odsh_extract_value_from_extras': odsh_helpers.odsh_extract_value_from_extras,
                    'odsh_create_checksum': odsh_helpers.odsh_create_checksum,
                    'presorted_license_options': odsh_helpers.presorted_license_options,
                    'odsh_has_more_facets': odsh_helpers.odsh_has_more_facets,
                    'odsh_public_url': odsh_helpers.odsh_public_url,
                    'odsh_spatial_extends_available': odsh_helpers.spatial_extends_available,
                    'odsh_public_resource_url': odsh_helpers.odsh_public_resource_url,
                    'odsh_get_version_id': odsh_helpers.odsh_get_version_id,
                    'odsh_show_testbanner': odsh_helpers.odsh_show_testbanner,
                    'odsh_is_slave': odsh_helpers.odsh_is_slave,
                    'tpsh_get_daterange_prettified': helper_pkg_dict.get_daterange_prettified,
                    'tpsh_get_language_of_package': helpers_tpsh.get_language_of_package,
                    'get_language_icon': helpers_tpsh.get_language_icon,
                    'short_name_for_category': odsh_helpers.short_name_for_category,
                    'get_spatial_for_selection': helpers_tpsh.get_spatial_for_selection,
                    'get_subject_for_selection': helpers_tpsh.get_subject_for_selection,
                    'get_language_for_selection': helpers_tpsh.get_language_for_selection,
                    'tpsh_get_resource_size': helpers_tpsh.get_resource_size,
                    'tpsh_get_address_org':helpers_tpsh.get_address_org,
                    'tpsh_get_body_mail':helpers_tpsh.get_body_mail,
    
                    'odsh_load_mdk_sample_dataset': odsh_helpers.odsh_load_mdk_sample_dataset,
    
                    'odsh_load_raw_mdk_sample_dataset': odsh_helpers.odsh_load_raw_mdk_sample_dataset,
    
    root's avatar
    root committed
        
        # IValidators
    
        def get_validators(self):
    
    anonymous's avatar
    anonymous committed
            return validation.get_validators()
    
    
        # Add the custom parameters to Solr's facet queries
    
        # use several daterange queries agains temporal_start and temporal_end field
        # TODO: use field of type date_range in solr index instead
    
        def before_search(self, search_params):
    
    root's avatar
    root committed
            return search.before_search(search_params)
    
    anonymous's avatar
    anonymous committed
        scores = [['0OL'], ['0OL', '1RE'], ['0OL', '1RE', '2OF'], [
            '0OL', '1RE', '2OF', '3URI'], ['0OL', '1RE', '2OF', '3URI', '4LD']]
    
    
        def map_qa_score(self, dict_pkg):
            if 'validated_data_dict' in dict_pkg and 'openness_score' in dict_pkg['validated_data_dict']:
    
    anonymous's avatar
    anonymous committed
                d = json.loads(dict_pkg['validated_data_dict'])
                score = -1
                for r in d['resources']:
                    if 'qa' in r:
    
                        if 'openness_score' in r['qa']:
                            s = int(r['qa']['openness_score'])
                            if s > score:
                                score = s
    
    anonymous's avatar
    anonymous committed
                if score > 0:
                    dict_pkg['openness'] = OdshPlugin.scores[score-1]