import datetime
import json
import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
from ckan.lib.plugins import DefaultTranslation
from ckan.lib.plugins import DefaultDatasetForm
from ckan.logic.validators import tag_string_convert
from ckan.logic.schema import default_extras_schema
from ckan.common import OrderedDict
from ckanext.odsh.lib.uploader import ODSHResourceUpload
import ckan.lib.helpers as helpers
import helpers as odsh_helpers
import ckanext.odsh.logic.action as action
from ckanext.dcat.interfaces import IDCATRDFHarvester
from ckanext.dcatde.extras import Extras
from routes.mapper import SubMapper
from pylons import config
from dateutil.parser import parse
from ckan import model
import ckan.plugins as p
import logging
import validation
import precondition
import sys
log = logging.getLogger(__name__)
# from functools import wraps
# from flask import Flask, redirect, jsonify
# app = Flask(__name__)
# def get_http_exception_handler(app):
# """Overrides the default http exception handler to return JSON."""
# handle_http_exception = app.handle_http_exception
# @wraps(handle_http_exception)
# def ret_val(exception):
# print("HEHREHR")
# exc = handle_http_exception(exception)
# return jsonify({'code':exc.code, 'message':exc.description}), exc.code
# return ret_val
# # Override the HTTP exception handler.
# app.handle_http_exception = get_http_exception_handler(app)
# def my_except_hook(exctype, value, traceback):
# print('GOT excepton')
# log.exception(value)
# sys.__excepthook__(exctype, value, traceback)
# print('INSTALL EX')
# sys.excepthook = my_except_hook
_ = toolkit._
from multiline_formatter.formatter import MultilineMessagesFormatter
class OdshLogger(MultilineMessagesFormatter):
multiline_marker = '...'
multiline_fmt = multiline_marker + ' : %(message)s'
def format(self, record):
"""
This is mostly the same as logging.Formatter.format except for the splitlines() thing.
This is done so (copied the code) to not make logging a bottleneck. It's not lots of code
after all, and it's pretty straightforward.
"""
endl_marker = '\n... : ";'
record.message = record.getMessage()
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
if '\n' in record.message:
splitted = record.message.splitlines()
output = self._fmt % dict(record.__dict__, message=splitted.pop(0))
output += ' ' + self.multiline_marker % record.__dict__ + '\n'
output += '\n'.join(
self.multiline_fmt % dict(record.__dict__, message=line)
for line in splitted
)
output = output.replace('"','\\"')
output += endl_marker
else:
output = self._fmt % record.__dict__
if record.exc_info:
# Cache the traceback text to avoid converting it multiple times
# (it's constant anyway)
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
output += ' ' + self.multiline_marker % record.__dict__ + '\n'
try:
output += '\n'.join(
self.multiline_fmt % dict(record.__dict__, message=line)
for index, line in enumerate(record.exc_text.splitlines())
)
output = output.replace('"','\\"')
output += endl_marker
except UnicodeError:
output += '\n'.join(
self.multiline_fmt % dict(record.__dict__, message=line)
for index, line
in enumerate(record.exc_text.decode(sys.getfilesystemencoding(), 'replace').splitlines())
)
return output
def odsh_get_facet_items_dict(name, limit=None):
'''
Gets all facets like 'get_facet_items_dict' but sorted alphabetically
instead by count.
'''
facets = helpers.get_facet_items_dict(name, limit)
facets.sort(key=lambda it: (it['display_name'].lower(), -it['count']))
return facets
def odsh_main_groups():
'''Return a list of the groups to be shown on the start page.'''
# Get a list of all the site's groups from CKAN, sorted by number of
# datasets.
groups = toolkit.get_action('group_list')(
data_dict={'all_fields': True})
return groups
def odsh_now():
return helpers.render_datetime(datetime.datetime.now(), "%Y-%m-%d")
def odsh_group_id_selected(selected, group_id):
if type(selected) is not list:
selected = [selected]
for g in selected:
if (isinstance(g, basestring) and group_id == g) or (type(g) is dict and group_id == g['id']):
return True
return False
def remove_route(map,routename):
route = None
for i,r in enumerate(map.matchlist):
if r.name == routename:
route = r
break
if route is not None:
map.matchlist.remove(route)
for key in map.maxkeys:
if key == route.maxkeys:
map.maxkeys.pop(key)
map._routenames.pop(route.name)
break
class OdshIcapPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IUploader, inherit=True)
def get_resource_uploader(self, data_dict):
return ODSHResourceUpload(data_dict)
class OdshAutocompletePlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IActions)
def get_actions(self):
return {'autocomplete': action.autocomplete}
class OdshHarvestPlugin(plugins.SingletonPlugin):
plugins.implements(plugins.IRoutes, inherit=True)
plugins.implements(plugins.IConfigurer)
def update_config(self, config_):
toolkit.add_template_directory(config_, 'harvest_templates')
plugins.implements(plugins.IRoutes, inherit=True)
def before_map(self, map):
DATASET_TYPE_NAME='harvest'
controller = 'ckanext.odsh.controller:OdshHarvestController'
map.connect('{0}_delete'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/delete/:id',controller=controller, action='delete')
map.connect('{0}_refresh'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/refresh/:id',controller=controller,
action='refresh')
map.connect('{0}_admin'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/admin/:id', controller=controller, action='admin')
map.connect('{0}_about'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/about/:id', controller=controller, action='about')
map.connect('{0}_clear'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/clear/:id', controller=controller, action='clear')
map.connect('harvest_job_list', '/' + DATASET_TYPE_NAME + '/{source}/job', controller=controller, action='list_jobs')
map.connect('harvest_job_show_last', '/' + DATASET_TYPE_NAME + '/{source}/job/last', controller=controller, action='show_last_job')
map.connect('harvest_job_show', '/' + DATASET_TYPE_NAME + '/{source}/job/{id}', controller=controller, action='show_job')
map.connect('harvest_job_abort', '/' + DATASET_TYPE_NAME + '/{source}/job/{id}/abort', controller=controller, action='abort_job')
map.connect('harvest_object_show', '/' + DATASET_TYPE_NAME + '/object/:id', controller=controller, action='show_object')
map.connect('harvest_object_for_dataset_show', '/dataset/harvest_object/:id', controller=controller, action='show_object', ref_type='dataset')
org_controller = 'ckanext.harvest.controllers.organization:OrganizationController'
map.connect('{0}_org_list'.format(DATASET_TYPE_NAME), '/organization/' + DATASET_TYPE_NAME + '/' + '{id}', controller=org_controller, action='source_list')
return map
def after_map(self, map):
return map
class OdshDCATHarvestPlugin(plugins.SingletonPlugin):
plugins.implements(IDCATRDFHarvester, inherit=True)
def before_update(self, harvest_object, dataset_dict, temp_dict):
existing_package_dict = self._get_existing_dataset(harvest_object.guid)
new_dataset_extras = Extras(dataset_dict['extras'])
if new_dataset_extras.key('modified') and \
new_dataset_extras.value('modified') < existing_package_dict.get('metadata_modified'):
log.info("Modified date of new dataset is not newer than "
+ "the already exisiting dataset, ignoring new one.")
dataset_dict.clear()
def _get_existing_dataset(self, guid):
'''
Checks if a dataset with a certain guid extra already exists
Returns a dict as the ones returned by package_show
'''
datasets = model.Session.query(model.Package.id) \
.join(model.PackageExtra) \
.filter(model.PackageExtra.key == 'guid') \
.filter(model.PackageExtra.value == guid) \
.filter(model.Package.state == 'active') \
.all()
if not datasets:
return None
elif len(datasets) > 1:
log.error('Found more than one dataset with the same guid: {0}'
.format(guid))
return p.toolkit.get_action('package_show')({}, {'id': datasets[0][0]})
class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm):
plugins.implements(plugins.IConfigurer)
plugins.implements(plugins.ITemplateHelpers)
plugins.implements(plugins.IRoutes, inherit=True)
plugins.implements(plugins.ITranslation)
plugins.implements(plugins.IFacets)
plugins.implements(plugins.IDatasetForm)
plugins.implements(plugins.IValidators)
plugins.implements(plugins.IPackageController, inherit=True)
plugins.implements(plugins.IActions)
# IActions
def get_actions(self):
return {'package_create': action.odsh_package_create,
'user_create': action.odsh_user_create}
# IConfigurer
def update_config(self, config_):
toolkit.add_template_directory(config_, 'templates')
toolkit.add_public_directory(config_, 'public')
toolkit.add_resource('fanstatic', 'odsh')
def get_helpers(self):
# Template helper function names should begin with the name of the
# extension they belong to, to avoid clashing with functions from
# other extensions.
return {'odsh_main_groups': odsh_main_groups,
'odsh_now': odsh_now,
'odsh_group_id_selected': odsh_group_id_selected,
'odsh_get_facet_items_dict': odsh_get_facet_items_dict,
'odsh_openness_score_dataset_html': odsh_helpers.odsh_openness_score_dataset_html,
'odsh_get_resource_details': odsh_helpers.odsh_get_resource_details,
'odsh_get_resource_views': odsh_helpers.odsh_get_resource_views,
'odsh_get_bounding_box': odsh_helpers.odsh_get_bounding_box,
'odsh_get_spatial_text': odsh_helpers.odsh_get_spatial_text,
'odsh_render_datetime': odsh_helpers.odsh_render_datetime,
'odsh_upload_known_formats': odsh_helpers.odsh_upload_known_formats,
'odsh_encodeurl': odsh_helpers.odsh_encodeurl,
'odsh_extract_error': odsh_helpers.odsh_extract_error,
'odsh_extract_error_new': odsh_helpers.odsh_extract_error_new,
'odsh_extract_value_from_extras': odsh_helpers.odsh_extract_value_from_extras,
'odsh_create_checksum': odsh_helpers.odsh_create_checksum,
'presorted_license_options': odsh_helpers.presorted_license_options,
'odsh_tracking_id': odsh_helpers.odsh_tracking_id,
'odsh_tracking_url': odsh_helpers.odsh_tracking_url,
'odsh_has_more_facets': odsh_helpers.odsh_has_more_facets,
'odsh_public_url': odsh_helpers.odsh_public_url,
'odsh_public_resource_url': odsh_helpers.odsh_public_resource_url
}
def after_map(self, map):
return map
def before_map(self, map):
map.connect('info_page', '/info_page',
controller='ckanext.odsh.controller:OdshRouteController', action='info_page')
map.connect('home', '/',
controller='ckanext.odsh.controller:OdshRouteController', action='start')
map.redirect('/dataset/{id}/resource/{resource_id}', '/dataset/{id}')
if p.toolkit.asbool(config.get('ckanext.dcat.enable_rdf_endpoints', True)):
remove_route(map, 'dcat_catalog')
map.connect('dcat_catalog',
config.get('ckanext.dcat.catalog_endpoint', '/catalog.{_format}'),
controller='ckanext.odsh.controller:OdshDCATController', action='read_catalog',
requirements={'_format': 'xml|rdf|n3|ttl|jsonld'})
# with SubMapper(map, controller='ckanext.odsh.controller:OdshApiController') as m:
# m.connect('/catalog2', action='read_catalog')
# /api ver 3 or none with matomo
GET_POST = dict(method=['GET', 'POST'])
with SubMapper(map, controller='ckanext.odsh.controller:OdshApiController', path_prefix='/api{ver:/3|}', ver='/3') as m:
m.connect('/action/{logic_function}', action='action', conditions=GET_POST)
with SubMapper(map, controller='ckanext.odsh.controller:OdshFeedController') as m:
m.connect('/feeds/custom.atom', action='custom')
with SubMapper(map, controller='ckanext.odsh.controller:OdshPackageController') as m:
m.connect('new_view', '/dataset/{id}/resource/{resource_id}/new_view', action='edit_view', ckan_icon='pencil-square-o')
with SubMapper(map, controller='ckanext.odsh.controller:OdshGroupController') as m:
m.connect('organizations_index', '/organization', action='index')
# redirect all user routes to custom controller
with SubMapper(map, controller='ckanext.odsh.controller:OdshUserController') as m:
m.connect('user_index', '/user', action='index')
m.connect('/user/edit', action='edit')
m.connect('user_edit', '/user/edit/{id:.*}', action='edit', ckan_icon='cog')
m.connect('user_delete', '/user/delete/{id}', action='delete')
m.connect('/user/reset/{id:.*}', action='perform_reset')
m.connect('/user/reset', action='request_reset')
m.connect('register', '/user/register', action='register')
m.connect('login', '/user/login', action='login')
m.connect('/user/_logout', action='logout')
m.connect('/user/logged_in', action='logged_in')
m.connect('/user/logged_out', action='logged_out')
m.connect('/user/logged_out_redirect', action='logged_out_page')
m.connect('user_datasets', '/user/{id:.*}', action='read',
ckan_icon='sitemap')
return map
def dataset_facets(self, facets_dict, package_type):
return OrderedDict({'organization': _('Herausgeber'),
'res_format': _('Dateiformat'),
'license_title': _('Lizenz'),
'groups': _('Kategorie'),
'openness': _('Open-Data-Eigenschaften')})
def organization_facets(self, facets_dict, organization_type, package_type):
return OrderedDict({'organization': _('Herausgeber'),
'res_format': _('Dateiformat'),
'license_title': _('Lizenz'),
'groups': _('Kategorie')})
def group_facets(self, facets_dict, group_type, package_type):
return OrderedDict({'organization': _('Herausgeber'),
'res_format': _('Dateiformat'),
'license_title': _('Lizenz'),
'groups': _('Kategorie')})
def _update_schema(self, schema):
for field in ['title', 'notes','license_id']:
schema.update({field: [toolkit.get_converter('not_empty')]})
for i, item in enumerate(schema['tags']['name']):
if item == toolkit.get_validator('tag_name_validator'):
schema['tags']['name'][i] = toolkit.get_validator(
'odsh_tag_name_validator')
for i, item in enumerate(schema['tag_string']):
if item == tag_string_convert:
schema['tag_string'][i] = validation.tag_string_convert
schema['resources'].update({
'url': [toolkit.get_converter('not_empty')],
'format': [toolkit.get_converter('not_empty')],
})
schema['extras'].update({
'key': [
toolkit.get_converter('known_spatial_uri'),
toolkit.get_converter('validate_licenseAttributionByText'),
]
})
schema.update({'__extras': [toolkit.get_converter('odsh_validate_extras')] })
def create_package_schema(self):
schema = super(OdshPlugin, self).create_package_schema()
self._update_schema(schema)
return schema
def update_package_schema(self):
schema = super(OdshPlugin, self).update_package_schema()
self._update_schema(schema)
return schema
def show_package_schema(self):
schema = super(OdshPlugin, self).show_package_schema()
return schema
def is_fallback(self):
# Return True to register this plugin as the default handler for
# package types not handled by any other IDatasetForm plugin.
return True
def package_types(self):
# This plugin doesn't handle any special package types, it just
# registers itself as the default (above).
return []
def get_validators(self):
return validation.get_validators()
# Add the custom parameters to Solr's facet queries
# use several daterange queries agains temporal_start and temporal_end field
# TODO: use field of type date_range in solr index instead
def before_search(self, search_params):
search_params['facet.mincount']=0
extras = search_params.get('extras')
print(search_params)
if not extras:
# There are no extras in the search params, so do nothing.
return search_params
fq = search_params['fq']
start_date=None
end_date=None
try:
start_date = odsh_helpers.extend_search_convert_local_to_utc_timestamp(
extras.get('ext_startdate'))
end_date = odsh_helpers.extend_search_convert_local_to_utc_timestamp(
extras.get('ext_enddate'))
except:
return search_params
empty_range = start_date and end_date and start_date > end_date
if not start_date and not end_date:
return search_params
do_enclosing_query = True
if not start_date:
do_enclosing_query = False
start_date = '*'
if not end_date:
do_enclosing_query = False
end_date = '*'
start_query = '+extras_temporal_start:[{start_date} TO {end_date}]'.format(
start_date=start_date, end_date=end_date)
end_query = '+extras_temporal_end:[{start_date} TO {end_date}]'.format(
start_date=start_date, end_date=end_date)
enclosing_query = ''
if do_enclosing_query and not empty_range:
enclosing_query_start = 'extras_temporal_start:[* TO {start_date}]'.format(
start_date=start_date)
enclosing_query_end = 'extras_temporal_end:[{end_date} TO *]'.format(
end_date=end_date)
enclosing_query = ' OR ({enclosing_query_start} AND {enclosing_query_end})'.format(
enclosing_query_start=enclosing_query_start, enclosing_query_end=enclosing_query_end)
if end_date is '*':
open_end_query = '(*:* NOT extras_temporal_end:[* TO *])'
else:
open_end_query = '((*:* NOT extras_temporal_end:[* TO *]) AND extras_temporal_start:[* TO {end_date}])'.format(
end_date=end_date)
fq = u'{fq} ({start_query} OR {end_query} {enclosing_query} OR {open_end_query})'.format(
fq=fq, start_query=start_query, end_query=end_query, enclosing_query=enclosing_query, open_end_query=open_end_query)
search_params['fq'] = fq
return search_params
scores = [ ['0OL'], ['0OL','1RE'], ['0OL','1RE','2OF'], ['0OL','1RE','2OF','3URI'], ['0OL','1RE','2OF','3URI','4LD']]
def map_qa_score(self, dict_pkg):
if 'validated_data_dict' in dict_pkg and 'openness_score' in dict_pkg['validated_data_dict']:
d = json.loads(dict_pkg['validated_data_dict'])
score = -1
for r in d['resources']:
if 'qa' in r:
i = r['qa'].find('openness_score')
s = int(r['qa'][i+17])
if s > score:
score=s
if score > 0:
dict_pkg['openness']=OdshPlugin.scores[score-1]
#@precondition.not_on_slave
def before_index(self, dict_pkg):
# make special date fields solr conform
fields = ["issued", "temporal_start", "temporal_end"]
for field in fields:
field = 'extras_' + field
if field in dict_pkg and dict_pkg[field]:
d = parse(dict_pkg[field])
dict_pkg[field] = '{0.year:04d}-{0.month:02d}-{0.day:02d}T00:00:00Z'.format(d)
# if 'res_format' in dict_pkg:
# dict_pkg['res_format']=[e.lower() for e in dict_pkg['res_format']]
self.map_qa_score(dict_pkg)
return dict_pkg