Merge branch 'dev' into 'master'

Stage System soll in Zukunft Master Branch erhalten

See merge request !17

ckanext/odsh/collection/controller.py

@@ -9,6 +9,8 @@ class LatestDatasetController(PackageController):
     
     def latest_dataset(self, id):
         latest_dataset= get_latest_dataset(id)
+        if latest_dataset is None:
+            toolkit.abort(404)
         toolkit.redirect_to(controller='package', action='read', id=latest_dataset)
 
 class LatestRecourcesController(PackageController):

ckanext/odsh/collection/helpers.py

@@ -10,13 +10,16 @@ def get_collection(dataset_dict):
     collection_id = get_collection_id(dataset_dict)
     if collection_id:
         return get_collection_info(collection_id, dataset_dict)
-    
     return None
 
 
 def get_collection_info(collection_id, dataset_dict=None):
     collection_dict = get_package_dict(collection_id)
+    if not collection_dict:
+        return None
     dataset_names = get_dataset_names(collection_dict)
+    if not dataset_names:
+        return None
     datasets_in_collection = get_datasets_from_solr(dataset_names)
     collection_info = gather_collection_info(collection_dict, datasets_in_collection, dataset_dict)
     return collection_info
@@ -41,6 +44,8 @@ def get_package_dict(name):
 
 
 def get_dataset_names(collection_dict):
+    if not collection_dict:
+        return []
     collection_dict = get_package_dict(collection_dict.get('id')) # needed to get full package_dict
     if collection_dict:
         relationships_collection = collection_dict.get('relationships')
@@ -53,6 +58,9 @@ def get_dataset_names(collection_dict):
 def get_datasets_from_solr(dataset_names):
     context = None
 
+    if not dataset_names:
+        return []
+
     name_expression = ' OR '.join(dataset_names)
     fq = 'name:({})'.format(name_expression)
     
@@ -73,6 +81,15 @@ def get_datasets_from_solr(dataset_names):
 
 
 def gather_collection_info(collection_dict, datasets_in_collection, dataset_dict=None):
+    url_collection = url_from_id(collection_dict.get('name'))
+
+    if not datasets_in_collection:
+        return {
+            'title': collection_dict.get('title'),
+            'url': url_collection,
+            'members': []
+        }
+
     name_first_dataset = datasets_in_collection[0].get('name')
     url_first_dataset = url_from_id(name_first_dataset)
     
@@ -82,7 +99,6 @@ def gather_collection_info(collection_dict, datasets_in_collection, dataset_dict
     name_collection = collection_dict.get('name')
     persistent_link_last_member = url_last_member(name_collection)
 
-    url_collection = url_from_id(collection_dict.get('name'))
 
     if dataset_dict:
         name_current_dataset = dataset_dict.get('name')
@@ -148,6 +164,8 @@ def url_last_member(name_collection):
 
 def get_latest_dataset(collection_name):
     collection_info = get_collection_info(collection_name)
+    if not collection_info:
+        return None
     latest_name = collection_info['last_member']['name']
     return latest_name
 
@@ -163,4 +181,6 @@ def get_latest_resources_for_format(collection_name, resource_format):
         return None
     resources_with_asked_type = [r for r in resources if r.get('format').upper() == resource_format.upper()]
     resources_sorted = sorted(resources_with_asked_type, key=itemgetter('id','created'), reverse=True)
+    if len(resources_sorted) < 2:
+        return None
     return resources_sorted[-1]

ckanext/odsh/controller.py

@@ -34,7 +34,7 @@ class OdshRouteController(HomeController):
         h.redirect_to('https://www.schleswig-holstein.de/odpinfo')
 
     def start(self):
-        h.redirect_to('https://opendata-stage.schleswig-holstein.de/dataset')
+        h.redirect_to('/dataset')
 
     def not_found(self):
         abort(404)

ckanext/odsh/fanstatic/odsh_populate_tags.js

+$(document).ready(function ()
+{
+    window.fieldTags = $("#field-tags").val();
+    $('#reference').on('change', function(e) {
+        const select = e.target;
+        const selectedOption = select.options[select.selectedIndex];
+        const tag = selectedOption.innerHTML.trim();
+        if(tag !== "Musterdatensatz wählen..") {
+            var tags = tag.split(' - ').join(',');
+            $("#field-tags").val(function() {
+                return window.fieldTags + `,${tags}`;
+            })
+        }
+    });
+});
+

ckanext/odsh/helpers.py

@@ -23,6 +23,8 @@ import pdb
 from urlparse import urlsplit, urlunsplit
 import subprocess
 import ckan.lib.helpers as helpers
+import os.path
+from collections import OrderedDict
 
 get_action = logic.get_action
 log = logging.getLogger(__name__)
@@ -92,6 +94,9 @@ def compute_bounding_box(coords):
     if len(coords) == 0:
         return None
 
+    if type(coords[0]) != list:
+        return [coords[0], coords[0], coords[1], coords[1]]
+
     coords = [c for sublist in coords for c in sublist]
     if type(coords[0][0]) == list:
         # multipolygon
@@ -453,3 +458,37 @@ def short_name_for_category(category_name):
         'tech': u'Wissenschaft',
     }
     return translations.get(category_name)
+
+def odsh_load_mdk_sample_dataset():
+    '''
+    Load sample dataset (Musterkatalog/Musterdatensatz).
+    
+    See https://bertelsmannstift.github.io/Musterdatenkatalog/def/musterdatensatz.rdf
+    and corresponding mapping in mdk_mapping.json file.
+    '''
+
+    path = os.path.abspath(os.path.dirname(__file__))
+    default_sample_data_file_path = os.path.join(path, "../../mdk_mapping.json")
+    sample_data_file_path = config.get(
+        'ckanext.odsh.sample_data_file_path', default_sample_data_file_path)
+    
+    try:
+        with open(sample_data_file_path) as mapping_json:
+             MDK_MAPPING = json.loads(mapping_json.read(), object_pairs_hook=OrderedDict)
+             default = [{'value': u'Musterdatensatz wählen..', 'key': u''}]
+             mdk = [{'key': key, 'value': MDK_MAPPING[key]} for key in MDK_MAPPING]
+             result = default+mdk
+    except IOError as err:
+        log.error(
+            'Could not load sample dataset mapping file from {}'
+            .format(sample_data_file_path)
+        )
+        raise
+    except ValueError as err:
+        log.error(
+            'Could not convert sample dataset mapping file from json. \nSample dataset mapping file: {}'
+            .format(sample_data_file_path)
+        )
+        raise
+    return result
+

ckanext/odsh/helpers_tpsh.py

@@ -216,11 +216,3 @@ def get_body_mail(organization, package):
     mail_url = "URL: " +  url + "%0D%0A"  +  "%0D%0A" 
     message =  mail_titel + mail_document  +  mail_url + "Mein Kommentar:" +  "%0D%0A"    +  "%0D%0A"  +  "%0D%0A"  +  "%0D%0A" 
     return anrede + message
-
-def git_commit_hash():
-    current_dir = os.path.dirname(os.path.abspath(__file__))
-    try:
-        command = 'git log -n 1 --format=%H'
-    except:
-        return 'unknown'
-    return subprocess.check_output([command], shell=True, cwd=current_dir)
\ No newline at end of file

ckanext/odsh/matomo.py

@@ -30,4 +30,4 @@ def create_matomo_request(userId=None):
     piwiktracker.set_api_url(config.get('ckanext.odsh.matomo_url'))
     if userId:
         piwiktracker.set_visitor_id(userId)
-    enqueue_job(piwiktracker.do_track_page_view,[request.path_qs], queue='tracking')
+    # enqueue_job(piwiktracker.do_track_page_view,[request.path_qs], queue='tracking')

ckanext/odsh/plugin.py

@@ -96,6 +96,13 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         for field in ['title', 'license_id']:
             schema.update({field: [toolkit.get_converter('not_empty')]})
         
+        schema.update({
+            'reference': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_converter('convert_to_extras')
+            ]
+        })
+        
         for i, item in enumerate(schema['tags']['name']):
             if item == toolkit.get_validator('tag_name_validator'):
                 schema['tags']['name'][i] = toolkit.get_validator(
@@ -106,7 +113,7 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
 
         schema['resources'].update({
             'url': [toolkit.get_converter('not_empty')],
-            'format': [toolkit.get_converter('not_empty')],
+            'format': [toolkit.get_converter('odsh_validate_format')],
         })
 
         schema['extras'].update({
@@ -374,7 +381,7 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 'tpsh_get_resource_size': helpers_tpsh.get_resource_size,
                 'tpsh_get_address_org':helpers_tpsh.get_address_org,
                 'tpsh_get_body_mail':helpers_tpsh.get_body_mail,
-		'tpsh_git_commit_hash': helpers_tpsh.git_commit_hash,
+                'odsh_load_mdk_sample_dataset': odsh_helpers.odsh_load_mdk_sample_dataset,
          }
 
     

ckanext/odsh/profiles/odsh_dcat_de_profile.py

@@ -8,6 +8,7 @@ from ckanext.dcat.utils import resource_uri
 import ckanext.dcatde.dataset_utils as ds_utils
 from ckanext.dcatde.profiles import DCATdeProfile, DCATDE, DCAT, DCATDE_1_0, DCATDE_1_0_1, DCATDE_1_0_2
 
+import ckanext.odsh.helpers as odsh_helpers
 import ckanext.odsh.helpers_tpsh as helpers_tpsh
 import ckanext.odsh.collection.helpers as helpers_collection
 
@@ -57,7 +58,6 @@ class ODSHDCATdeProfile(DCATdeProfile):
     def _mark_for_adding_to_ckan_collection(self, dataset_dict, dataset_ref):
         dataset_dict.update({'add_to_collection': True})
 
-    
     # to RDF
 
     def graph_from_dataset(self, dataset_dict, dataset_ref):
@@ -72,6 +72,8 @@ class ODSHDCATdeProfile(DCATdeProfile):
         self._add_contributor_id(dataset_dict, dataset_ref)
         self._add_license_attribution_by_text(dataset_dict, dataset_ref)
         self._add_type(dataset_dict, dataset_ref)
+        self._add_modified_and_issued(dataset_dict, dataset_ref)
+        self._add_references(dataset_dict, dataset_ref)
         if self._is_dataset_collection(dataset_dict):
             self._remove_predefined_collection_members()
             self._add_collection_members(dataset_dict, dataset_ref)
@@ -87,14 +89,52 @@ class ODSHDCATdeProfile(DCATdeProfile):
         )
 
     def _add_license_attribution_by_text(self, dataset_dict, dataset_ref):
-        licenseAttributionByText = self._get_dataset_value(dataset_dict, 'licenseAttributionByText')
+        licenseAttributionByText = self._get_dataset_value(
+            dataset_dict, 'licenseAttributionByText')
         if licenseAttributionByText:
             self.g.set(
-                (dataset_ref, DCATDE.licenseAttributionByText, rdflib.Literal(licenseAttributionByText))
+                (dataset_ref, DCATDE.licenseAttributionByText,
+                 rdflib.Literal(licenseAttributionByText))
+            )
+            for distribution in self.g.objects(dataset_ref, DCAT.distribution):
+                self.g.set(
+                    (distribution, DCATDE.licenseAttributionByText,
+                     rdflib.Literal(licenseAttributionByText))
+                )
+
+    def _add_references(self, dataset_dict, dataset_ref):
+        '''
+        Adds reference (Musterdatenkatalog/Musterdatensatz) extra field to
+        dcat:references.
+        '''
+        sample_dataset_uri = odsh_helpers.odsh_extract_value_from_extras(dataset_dict.get('extras'), 'reference')
+        if sample_dataset_uri:
+            self.g.set(
+                (dataset_ref, DCT.references,
+                    rdflib.URIRef(sample_dataset_uri)
                  )
+            )
+    
+    def _add_modified_and_issued(self, dataset_dict, dataset_ref):
+        '''
+        Adds distributions last_modified and created values to
+        dcat:modified and dcat:issued.
+        '''
         for distribution in self.g.objects(dataset_ref, DCAT.distribution):
+            for resource_dict in dataset_dict.get('resources', []):
+                # Match distribution in graph and distribution in ckan-dict
+                if unicode(distribution) == resource_uri(resource_dict):
+                    last_modified = resource_dict.get('last_modified', None)
+                    if last_modified:
+                        self.g.set(
+                            (distribution, DCT.modified, rdflib.Literal(
+                                last_modified, datatype="http://www.w3.org/2001/XMLSchema#dateTime"))
+                        )
+                    created = resource_dict.get('created', None)
+                    if created:
                         self.g.set(
-                    (distribution, DCATDE.licenseAttributionByText, rdflib.Literal(licenseAttributionByText))
+                            (distribution, DCT.issued, rdflib.Literal(
+                                created, datatype="http://www.w3.org/2001/XMLSchema#dateTime"))
                         )
 
     def _add_type(self, dataset_dict, dataset_ref):
@@ -121,7 +161,8 @@ class ODSHDCATdeProfile(DCATdeProfile):
                 self.g.remove((s, p, o))
 
     def _add_collection_members(self, dataset_dict, dataset_ref):
-        dataset_refs_belonging_to_collection = self._get_dataset_refs_belonging_to_collection(dataset_dict)
+        dataset_refs_belonging_to_collection = self._get_dataset_refs_belonging_to_collection(
+            dataset_dict)
         for ref in dataset_refs_belonging_to_collection:
             self.g.add(
                 (dataset_ref, DCT.hasVersion, rdflib.URIRef(ref))
@@ -134,8 +175,10 @@ class ODSHDCATdeProfile(DCATdeProfile):
 
     def _get_dataset_refs_belonging_to_collection(self, dataset_dict):
         dataset_names = helpers_collection.get_dataset_names(dataset_dict)
-	dataset_dicts = [model.Package.get(name).as_dict() for name in dataset_names]
-        dataset_ids = [dataset_dict.get('id') for dataset_dict in dataset_dicts]
+        dataset_dicts = [model.Package.get(
+            name).as_dict() for name in dataset_names]
+        dataset_ids = [dataset_dict.get('id')
+                       for dataset_dict in dataset_dicts]
         dataset_refs = [self._construct_refs(id) for id in dataset_ids]
         return dataset_refs
 

ckanext/odsh/public/odsh.css

@@ -824,6 +824,9 @@ a:hover.organization-item ,a:focus.organization-item
 
 @media (max-width: 719px)
 {
+.odsh-dataset-item {
+    flex-direction: column;
+}
 .dataset-content {
     width: 100%; 
     display: block;
@@ -831,6 +834,7 @@ a:hover.organization-item ,a:focus.organization-item
 .dataset-meta {
     width: 100%;
     display: block;
+    margin-top: 25px;
 }
 .dataset-spacer{
     display: none;
@@ -1329,7 +1333,6 @@ body {
         box-sizing: border-box;
         padding-bottom: 15px;
         background-image: none;
-        height: 150px;
         display: flex;
         flex-direction: column-reverse;
     }
@@ -1340,6 +1343,7 @@ body {
     .footer-right{
         display: flex;
         justify-content: space-between;
+        flex-direction: column;
         padding-bottom: 15px;
     }
     .footer-icon a{
@@ -1934,7 +1938,8 @@ body {
 }
 
 .field-organization .select2-container .select2-choice,
-.field-spatial_uri .select2-container .select2-choice 
+.field-spatial_uri .select2-container .select2-choice,
+.field-reference .select2-container .select2-choice 
 {
     background-color: #F6F7F9;
     background-image: none;

ckanext/odsh/public/odsh_header.css

@@ -38,7 +38,7 @@
     padding-right: 0;
     position: relative;
     top: 0;
-    left: 18px;
+    margin-left: 18px;
     padding: 1em 0;
 }
 

ckanext/odsh/templates/base.html

@@ -19,18 +19,21 @@
 
 {% if h.odsh_use_matomo() %}
 <!-- Matomo -->
-<script type="text/javascript">
-  var _paq = _paq || [];
+<script>
+  var _paq = window._paq = window._paq || [];
   /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+  _paq.push(["setExcludedQueryParams", ["zeile","cms_gtp","cms_range","kurzenachricht","zeig","projekt","datum","nn"]]);
   _paq.push(['trackPageView']);
   _paq.push(['enableLinkTracking']);
   (function() {
-    _paq.push(['setTrackerUrl', '{{matomo_url}}']);
-    _paq.push(['setSiteId', '{{matomo_id}}']);
+      var u="https://landesportal-sh.dwebanalytics.de/";
+      _paq.push(['setTrackerUrl', u+'matomo.php']);
+      _paq.push(['setSiteId', '3']);
       var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
-    g.type='text/javascript'; g.async=true; g.defer=true; g.src='{{matomo_url|replace('piwik.php','piwik.js')}}'; s.parentNode.insertBefore(g,s);
+      g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
     })();
 </script>
+<noscript><p><img src="https://landesportal-sh.dwebanalytics.de/matomo.php?idsite=3&amp;rec=1" style="border:0;" alt="" /></p></noscript>
 <!-- End Matomo Code -->
 {% endif %}
 {% endblock %}

ckanext/odsh/templates/header.html

@@ -44,7 +44,6 @@
                     {% block header_site_navigation_tabs %}
                     {{ 
                     h.build_nav_main(
-                        ('home', _('Start')),
                         ('search', _('Datensätze')),
                         ('organizations_index', _('Herausgeber')),
                         ('info_page', _('Infos'))

ckanext/odsh/templates/macros/form.html

@@ -366,12 +366,17 @@ is_required=is_required) %}
 {% macro input_address(field, label, value='', index='', placeholder='', type='text', attrs={}) %}
 {%- set _type = 'text' if type=='date' and not value else type -%}
 {%- set onFocus = 'onfocus=(this.type=\'date\')' if type=='date' and not value else '' -%}
+<div class="control-group control-full">
+	<label class="control-label" for="field-{{field}}-value">{{ label }}</label>
+	<div class="controls editor">
 		<div class="row-fluid">
 			<div class="span6">
-    <label class="address-label"> {{ label }} </label>
 				<input id="field-{{field}}-key" type="hidden" name="extras__{{index}}__key" value="{{field}}"  />
 				<input id="field-{{field}}-value" type="{{_type}}" name="extras__{{index}}__value" value="{{value | empty_and_escape }}" placeholder="{{ placeholder }}" {{ onFocus }} {{ attributes(attrs) }}/>
 			</div>
+			<div class="span6 inline-error"></div>
+		</div>
+	</div>
 </div>
 {% endmacro %}
 

ckanext/odsh/templates/organization/snippets/organization_form.html

@@ -37,12 +37,14 @@
   
   {{ form.input_address('location','Stadt', value=extras.location, index=2, placeholder='', type='text', attrs={}) }}
   
-  {{ form.input_address('telephone','Tel.-Nr.:', value=extras.telephone, index=3, placeholder='', type='text', attrs={}) }}
+  {{ form.input_address('telephone','Tel.-Nr.', value=extras.telephone, index=3, placeholder='', type='text', attrs={}) }}
 
-  {{ form.input_address('mail','e-mail', value=extras.mail, index=4, placeholder='', type='text', attrs={}) }}
+  {{ form.input_address('mail','E-Mail', value=extras.mail, index=4, placeholder='', type='text', attrs={}) }}
  
   {{ form.input_address('web','Website', value=extras.web, index=5, placeholder='', type='text', attrs={}) }}
 
+  {{ form.input_address('gnd','GND URI', value=extras.gnd, index=6, placeholder='https://d-nb.info/gnd/1136109587', type='text', attrs={}) }}
+
   {{ form.required_message() }}
 
   <div class="form-actions">

ckanext/odsh/templates/package/snippets/package_basic_fields.html

 {% import 'macros/form.html' as form %}
 {% resource 'odsh/odsh_form.js' %}
+{% resource 'odsh/odsh_populate_tags.js' %}
 {% resource 'odsh/bootstrap-multiselect.js' %}
 {% set dataset_is_draft = data.get('state', 'draft').startswith('draft') or data.get('state', 'none') == 'none' %}
 
@@ -320,3 +321,10 @@ dataset_is_draft)) %}
         </div>
     </div>
 </div>
+
+            
+{# field reference #}
+{% set field = 'reference' %}
+{% set error_reference = h.odsh_extract_error(field, errors) %}
+{% set value = h.odsh_extract_value_from_extras(data.extras,field) or '' %}
+{{ form.select_autocomplete(field, label=_('Musterdatensatz'), selected=value, options=h.odsh_load_mdk_sample_dataset(), error=error_reference, is_required=False, classes=['control-full', 'field-reference'])}}

ckanext/odsh/templates/package/snippets/resource_item.html

@@ -6,6 +6,7 @@
 
 {% set rtitle=h.resource_display_name(res) if res.name else ' '%}
 {% set resource_size = h.tpsh_get_resource_size(res) %}
+{% set res_format =  res.format.replace('_SRVC','') %}
 
 <li class="resource-item" data-id="{{ res.id }}">
     <div class="resource-title-container">
@@ -35,9 +36,9 @@
             {% endif %}
             {% endblock %}
         </div>
-            {% if res.format %}
+            {% if res_format %}
             <a href="{{ download }}" >
-               <div class="dataformat-label resource-dataformat-label label" style='font-size:{{150/(res.format|length)}}px'>{{res.format}}</div>
+               <div class="dataformat-label resource-dataformat-label label" style='font-size:{{150/(res_format|length)}}px'>{{res_format}}</div>
             </a>
             {% endif %}
     </div>

ckanext/odsh/templates/snippets/package_item.html

@@ -121,8 +121,8 @@ Example:
       {% for resource in h.dict_list_reduce(package.resources, 'format') %}
       <li>
         <a href="{{ h.url_for(controller='package', action='read', id=package.name) }}" class="label dataformat-label"
-          data-format="{{ resource.lower() }}">{{
-          resource }}</a>
+          data-format="{{ resource.lower().replace('_srvc','') }}">{{
+          resource.replace('_SRVC','') }}</a>
       </li>
       {% endfor %}
       {% endblock %}