Merge remote-tracking branch 'origin/master'

# Conflicts:
#	ckanext/odsh/controller.py

ckanext/odsh/controller.py

+from types import FunctionType
 import ckan.lib.base as base
+import decorator
 from ckan.controllers.home import HomeController
 from ckan.controllers.user import UserController
 from ckan.controllers.api import ApiController
+from ckan.controllers.group import GroupController
+from ckanext.harvest.controllers.view import ViewController as HarvestController
 from ckan.controllers.feed import FeedController
 from ckan.controllers.package import PackageController
 from ckan.controllers.feed import FeedController, ITEMS_LIMIT, _package_search, _create_atom_id
@@ -15,22 +19,31 @@ from ckan.common import c, request, config
 import hashlib
 import ckan.plugins.toolkit as toolkit
 from ckanext.dcat.controllers import DCATController
-
+import ckan.model as model
 
 abort = base.abort
 log = logging.getLogger(__name__)
+render = base.render
+get_action = logic.get_action
 
 
 class OdshRouteController(HomeController):
     def info_page(self):
         h.redirect_to('http://www.schleswig-holstein.de/odpinfo')
+
     def start(self):
         h.redirect_to('http://www.schleswig-holstein.de/odpstart')
+
     def not_found(self):
         abort(404)
 
 
 class OdshUserController(UserController):
+    def index(self):
+        if not authz.is_sysadmin(c.user):
+            abort(404)
+        return super(OdshUserController, self).index()
+
     def me(self, locale=None):
         if not c.user:
             h.redirect_to(locale=locale, controller='user', action='login',
@@ -49,6 +62,8 @@ class OdshUserController(UserController):
         return super(OdshUserController, self).dashboard_datasets(id)
 
     def read(self, id=None):
+        if not c.user:
+            h.redirect_to(controller='user', action='login')
         return super(OdshUserController, self).read(id)
 
     def follow(self, id):
@@ -66,9 +81,91 @@ class OdshUserController(UserController):
             abort(404)
         return super(OdshUserController, self).activity(id, offset)
 
+    def register(self, data=None, errors=None, error_summary=None):
+        if not authz.is_sysadmin(c.user):
+            abort(404)
+        return super(OdshUserController, self).register(data, errors, error_summary)
+
 
 class OdshPackageController(PackageController):
-    pass
+    def edit_view(self, id, resource_id, view_id=None):
+        if not authz.is_sysadmin(c.user):
+            abort(403)
+        return super(OdshPackageController, self).edit_view(id, resource_id, view_id)
+
+
+class OdshGroupController(GroupController):
+    def index(self):
+        group_type = self._guess_group_type()
+
+        page = h.get_page_number(request.params) or 1
+        items_per_page = 21
+
+        context = {'model': model, 'session': model.Session,
+                   'user': c.user, 'for_view': True,
+                   'with_private': False}
+
+        query = c.q = request.params.get('q', '')
+        sort_by = c.sort_by_selected = request.params.get('sort')
+        try:
+            self._check_access('site_read', context)
+            self._check_access('group_list', context)
+        except NotAuthorized:
+            abort(403, _('Not authorized to see this page'))
+
+        # pass user info to context as needed to view private datasets of
+        # orgs correctly
+        if c.userobj:
+            context['user_id'] = c.userobj.id
+            context['user_is_admin'] = c.userobj.sysadmin
+
+        for q in query.split(' '):
+            try:
+                data_dict_global_results = {
+                    'all_fields': False,
+                    'q': q,
+                    'sort': sort_by,
+                    'type': group_type or 'group',
+                }
+                print("QUERY")
+                print(group_type)
+                print(q)
+                global_results = self._action('group_list')(
+                    context, data_dict_global_results)
+            except ValidationError as e:
+                if e.error_dict and e.error_dict.get('message'):
+                    msg = e.error_dict['message']
+                else:
+                    msg = str(e)
+                h.flash_error(msg)
+                c.page = h.Page([], 0)
+                return render(self._index_template(group_type),
+                              extra_vars={'group_type': group_type})
+
+            data_dict_page_results = {
+                'all_fields': True,
+                'q': q,
+                'sort': sort_by,
+                'type': group_type or 'group',
+                'limit': items_per_page,
+                'offset': items_per_page * (page - 1),
+                'include_extras': True
+            }
+            page_results = self._action('group_list')(context,
+                                                      data_dict_page_results)
+
+        print("GROUPS")
+        print(global_results)
+        c.page = h.Page(
+            collection=global_results,
+            page=page,
+            url=h.pager_url,
+            items_per_page=items_per_page,
+        )
+
+        c.page.items = page_results
+        return render(self._index_template(group_type),
+                      extra_vars={'group_type': group_type})
 
 
 class OdshApiController(ApiController):
@@ -106,7 +203,8 @@ class OdshDCATController(DCATController):
 class OdshFeedController(FeedController):
     def custom(self):
         matomo.create_matomo_request()
-        extra_fields=['ext_startdate', 'ext_enddate', 'ext_bbox', 'ext_prev_extent']
+        extra_fields = ['ext_startdate', 'ext_enddate',
+                        'ext_bbox', 'ext_prev_extent']
         q = request.params.get('q', u'')
         fq = ''
         search_params = {}
@@ -160,3 +258,25 @@ class OdshFeedController(FeedController):
                                 feed_guid=_create_atom_id(atom_url),
                                 feed_url=feed_url,
                                 navigation_urls=navigation_urls)
+
+
+def only_admin(func, *args, **kwargs):
+    if not authz.is_sysadmin(c.user):
+        abort(404)
+    return func(*args, **kwargs)
+
+
+class MetaClass(type):
+    def __new__(meta, classname, bases, classDict):
+        newClassDict = {}
+        wdec = decorator.decorator(only_admin)
+        for attributeName, attribute in bases[0].__dict__.items():
+             if isinstance(attribute, FunctionType) and not attributeName.startswith('_'):
+                 print(attribute)
+                 attribute = wdec(attribute)
+             newClassDict[attributeName] = attribute
+        return type.__new__(meta, classname, bases, newClassDict)
+
+
+class OdshHarvestController(HarvestController):
+    __metaclass__ = MetaClass  # wrap all the methods

ckanext/odsh/harvest_templates/source/search.html

+{# this template checks for sysadmin and shows a 404 if not. This is a hack as the harvest extension has no way for restricting access #}
+{% extends "page.html" %}
+
+{% block subtitle %}
+  {% if c.userobj.sysadmin %}
+    {{ _("Harvest sources") }}
+  {% else %}
+    {{ gettext('Error %(error_code)s', error_code=c.code[0]) }}
+  {% endif %}
+{% endblock %}
+
+
+{% block breadcrumb_content %}
+  <li class="active">{{ h.nav_link(_('Harvest Sources'), named_route='{0}_search'.format(c.dataset_type)) }}</li>
+{% endblock %}
+
+{% if g.ckan_base_version.startswith('2.0') %}
+  {# CKAN 2.0 #}
+
+  {% block add_action_content %}
+    {{ h.snippet('snippets/add_source_button.html', dataset_type=c.dataset_type) }}
+  {% endblock %}
+{% endif %}
+
+{% block primary_content %}
+  {% if c.userobj.sysadmin %}
+  {% if g.ckan_base_version.startswith('2.0') %}
+    {# CKAN 2.0 #}
+
+    {% include 'source/search_2.0.html' %}
+
+  {% else %}
+    {# > CKAN 2.0 #}
+
+    <section class="module">
+      <div class="module-content">
+      {#
+        {% block page_primary_action %}
+          <div class="page_primary_action">
+            {{ h.snippet('snippets/add_source_button.html', dataset_type=c.dataset_type) }}
+          </div>
+        {% endblock %}
+        {% set facets = {
+          'fields': c.fields_grouped,
+          'search': c.search_facets,
+          'titles': c.facet_titles,
+          'translated_fields': c.translated_fields,
+          'remove_field': c.remove_field }
+        %}
+        {% set sorting = [
+          (_('Relevance'), 'score desc, metadata_modified desc'),
+          (_('Name Ascending'), 'title_string asc'),
+          (_('Name Descending'), 'title_string desc'),
+          (_('Last Modified'), 'metadata_modified desc'),
+          (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false) ]
+        %}
+        {% snippet 'snippets/search_form.html', type='harvest', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, show_empty=request.params, error=c.query_error, placeholder=_("Search harvest sources...") %}
+      #}
+
+        {{ h.snippet('snippets/source_list.html', sources=c.page.items, show_organization=true) }}
+
+      </div>
+
+      {{ c.page.pager(q=c.q) }}
+    </section>
+
+  {% endif %}
+  {% else %}
+    <div class="module-content error-page">
+        <div class="error-title">
+            HTTP Status 404 
+                  <div class="error-body"><h2>Seite nicht gefunden</h2>
+                <h3>Wie finde ich die gesuchten Inhalte im Landesportal?</h3>
+
+                <p><a class="" href="http://www.schleswig-holstein.de/odpstart" title="Zur Startseite">Zur Startseite des Open-Data-Portals</a></p>
+
+                <h3>Kontakt</h3>
+                <p>Bei Fragen oder Problemen mit dem Open-Data-Portal schicken Sie bitte eine E-Mail an die Adresse opendata@lr.landsh.de oder verwenden das Kontaktformular:</p>
+                <p><a class="" href="https://www.schleswig-holstein.de/odpkontakt" title="Kontakt">Zum Kontaktformular</a></p>
+            </div>
+        </div>
+    </div>
+  {% endif %}
+
+{% endblock %}
+
+{% block breadcrumb %}
+{% endblock %}
+
+{% block secondary %}{% endblock %}
+{#
+{% block secondary_content %}
+  {% if c.userobj.sysadmin %}
+  {% for facet in c.facet_titles %}
+      {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, alternative_url=h.url_for('{0}_search'.format(c.dataset_type))) }}
+  {% endfor %}
+  {% endif %}
+{% endblock %}
+#}
\ No newline at end of file

ckanext/odsh/plugin.py

@@ -90,6 +90,39 @@ class OdshAutocompletePlugin(plugins.SingletonPlugin):
     def get_actions(self):
         return {'autocomplete': action.autocomplete}
 
+class OdshHarvestPlugin(plugins.SingletonPlugin):
+    plugins.implements(plugins.IRoutes, inherit=True)
+    plugins.implements(plugins.IConfigurer)
+
+    def update_config(self, config_):
+        toolkit.add_template_directory(config_, 'harvest_templates')
+    plugins.implements(plugins.IRoutes, inherit=True)
+    def before_map(self, map):
+        DATASET_TYPE_NAME='harvest'
+        controller = 'ckanext.odsh.controller:OdshHarvestController'
+
+        map.connect('{0}_delete'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/delete/:id',controller=controller, action='delete')
+        map.connect('{0}_refresh'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/refresh/:id',controller=controller,
+                action='refresh')
+        map.connect('{0}_admin'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/admin/:id', controller=controller, action='admin')
+        map.connect('{0}_about'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/about/:id', controller=controller, action='about')
+        map.connect('{0}_clear'.format(DATASET_TYPE_NAME), '/' + DATASET_TYPE_NAME + '/clear/:id', controller=controller, action='clear')
+
+        map.connect('harvest_job_list', '/' + DATASET_TYPE_NAME + '/{source}/job', controller=controller, action='list_jobs')
+        map.connect('harvest_job_show_last', '/' + DATASET_TYPE_NAME + '/{source}/job/last', controller=controller, action='show_last_job')
+        map.connect('harvest_job_show', '/' + DATASET_TYPE_NAME + '/{source}/job/{id}', controller=controller, action='show_job')
+        map.connect('harvest_job_abort', '/' + DATASET_TYPE_NAME + '/{source}/job/{id}/abort', controller=controller, action='abort_job')
+
+        map.connect('harvest_object_show', '/' + DATASET_TYPE_NAME + '/object/:id', controller=controller, action='show_object')
+        map.connect('harvest_object_for_dataset_show', '/dataset/harvest_object/:id', controller=controller, action='show_object', ref_type='dataset')
+
+        org_controller = 'ckanext.harvest.controllers.organization:OrganizationController'
+        map.connect('{0}_org_list'.format(DATASET_TYPE_NAME), '/organization/' + DATASET_TYPE_NAME + '/' + '{id}', controller=org_controller, action='source_list')
+        return map
+
+    def after_map(self, map):
+        return map
+
 
 class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm):
     plugins.implements(plugins.IConfigurer)
@@ -171,8 +204,18 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         with SubMapper(map, controller='ckanext.odsh.controller:OdshFeedController') as m:
             m.connect('/feeds/custom.atom', action='custom')
 
+        # with SubMapper(map, controller='ckanext.odsh.controller:OdshHarvestController') as m:
+        #     m.connect('/harvest', action='index')
+
+        with SubMapper(map, controller='ckanext.odsh.controller:OdshPackageController') as m:
+            m.connect('new_view', '/dataset/{id}/resource/{resource_id}/new_view', action='edit_view', ckan_icon='pencil-square-o')
+
+        # with SubMapper(map, controller='ckanext.odsh.controller:OdshGroupController') as m:
+        #     m.connect('organizations_index', '/organization', action='index')
+
         # redirect all user routes to custom controller
         with SubMapper(map, controller='ckanext.odsh.controller:OdshUserController') as m:
+            m.connect('user_index', '/user', action='index')
             m.connect('/user/edit', action='edit')
             m.connect('user_edit', '/user/edit/{id:.*}', action='edit', ckan_icon='cog')
             m.connect('user_delete', '/user/delete/{id}', action='delete')

ckanext/odsh/templates/error_document_template.html

@@ -7,8 +7,12 @@
     <div class="module-content error-page">
         <div class="error-title">
             HTTP Status {{ c.code[0]}}
+            {%if c.code[0]=='404' or c.code[0]=='403'%}
                 {%if c.code[0]=='404'%}
                   <div class="error-body"><h2>Seite nicht gefunden</h2>
+                {%elif c.code[0]=='403'%}
+                  <div class="error-body"><h2>Zugriff nicht erlaubt</h2>
+                {% endif %}
                 <h3>Wie finde ich die gesuchten Inhalte im Landesportal?</h3>
 
                 <p><a class="" href="http://www.schleswig-holstein.de/odpstart" title="Zur Startseite">Zur Startseite des Open-Data-Portals</a></p>

ckanext/odsh/tests/test_validation.py

+import sys
+import json
+from nose.tools import *
+from mock import MagicMock, Mock, patch
+
+
+def mockInvalid(*args, **kwargs):
+    return Exception(*args, **kwargs)
+
+def mock_(s):
+    return s
+
+m = MagicMock()
+class MissingMock:
+    pass
+m.Missing=MissingMock
+
+sys.modules['ckan'] = MagicMock()
+sys.modules['ckan.plugins'] = MagicMock()
+sys.modules['ckan.plugins.toolkit'] = MagicMock()
+sys.modules['ckan.model'] = MagicMock()
+sys.modules['ckan.lib'] = MagicMock()
+sys.modules['ckan.lib.navl'] = MagicMock()
+sys.modules['ckan.lib.navl.dictization_functions'] = m
+sys.modules['pylons'] = MagicMock()
+
+import ckan.model as modelMock
+import pylons
+import ckan.plugins.toolkit as toolkit
+
+toolkit.Invalid = mockInvalid
+toolkit._ = mock_
+
+
+from ckanext.odsh.validation import *
+
+
+def test_get_validators():
+    assert get_validators()
+
+
+def test_tag_string_convert():
+    # arrange
+    data = {'tag_string': 'tag1,tag2'}
+    # act
+    tag_string_convert('tag_string', data, {}, None)
+    # assert
+    assert data[('tags', 0, 'name')] == 'tag1'
+    assert data[('tags', 1, 'name')] == 'tag2'
+
+
+@raises(Exception)
+def test_tag_name_validator_invalid():
+    tag_name_validator('&', None)
+
+
+def test_tag_name_validator_valid():
+    tag_name_validator('valid', None)
+
+
+@patch('urllib2.urlopen')
+@patch('pylons.config.get', side_effect='foo')
+@patch('csv.reader', side_effect=[[['uri', 'text', json.dumps({"geometry": 0})]]])
+def test_known_spatial_uri(url_mock, get_mock, csv_mock):
+    # arrange
+    data = {('extras', 0, 'key'): 'spatial_uri',
+            ('extras', 0, 'value'): 'uri'}
+    # act
+    known_spatial_uri('spatial_uri', data, {}, None)
+    # assert
+    assert data[('extras', 1, 'key')] == 'spatial_text'
+    assert data[('extras', 1, 'value')] == 'text'
+    assert data[('extras', 2, 'key')] == 'spatial'
+    assert data[('extras', 2, 'value')] == '0'
+
+
+def test_validate_licenseAttributionByText():
+    # arrange
+    def get_licenses():
+        return {}
+    modelMock.Package.get_license_register = get_licenses
+    data = {'license_id': '0',
+            ('extras', 0, 'key'): 'licenseAttributionByText',
+            ('extras', 0, 'value'): ''}
+    validate_licenseAttributionByText('key', data, {}, None)

ckanext/odsh/validation.py

@@ -104,6 +104,7 @@ def validate_licenseAttributionByText(key, data, errors,context):
             isByLicense = True
             break
     hasAttribution=False
+    print(Missing)
     for k in data:
         if data[k] == 'licenseAttributionByText':
             if isinstance(data[(k[0], k[1], 'value')], Missing) or (k[0], k[1], 'value') not in data:

setup.py

@@ -85,6 +85,7 @@ setup(
         statistikamtnord_harvester=ckanext.odsh.harvesters:StatistikamtNordHarvester
         kiel_harvester=ckanext.odsh.harvesters:KielHarvester
         odsh_autocomplete=ckanext.odsh.plugin:OdshAutocompletePlugin
+        odsh_harvest=ckanext.odsh.plugin:OdshHarvestPlugin
 
         [paste.paster_command]
         odsh_initialization = ckanext.odsh.commands.initialization:Initialization