disabled thumbnail from url

ckanext/odsh/lib/uploader.py

@@ -23,7 +23,7 @@ def _icap_virus_found(filename, upload_file):
         return response_object.virus_found()
 
 
-def _raise_validation_error_if_virus_found(filename, upload_file):
+def raise_validation_error_if_virus_found(filename, upload_file):
     if _icap_virus_found(filename, upload_file):
         raise logic.ValidationError({'upload': ['Virus gefunden']})
 
@@ -50,7 +50,7 @@ class ODSHResourceUpload(ResourceUpload):
         log.debug("Resource({}) uploaded.".format(resource))
         super(ODSHResourceUpload, self).__init__(resource)
         if hasattr(self, 'filename') and hasattr(self, 'upload_file'):
-            _raise_validation_error_if_virus_found(self.filename, self.upload_file)
+            raise_validation_error_if_virus_found(self.filename, self.upload_file)
             _raise_validation_error_if_hash_values_differ(self.upload_file, resource)
 
 
@@ -70,5 +70,5 @@ class ODSHUpload(Upload):
     
     def upload(self, max_size=2):
         if hasattr(self, 'filename') and hasattr(self, 'upload_file'):
-            _raise_validation_error_if_virus_found(self.filename, self.upload_file)
+            raise_validation_error_if_virus_found(self.filename, self.upload_file)
         super(ODSHUpload, self).upload(max_size)
\ No newline at end of file

ckanext/odsh/pdf_to_thumbnail/action.py

+# ckan
+import ckan.model as model
+import ckan.logic as logic
+import ckan.lib.helpers as helpers
+import ckan.model as model
+from ckan.logic.action.update import package_update
+from ckan.logic.action.delete import package_delete
+
+#from thumbnail
+import thumbnail as thumbnail
+
+import os as os
+
+
+def before_package_delete(context, package_id_dict):
+    pkg_dict = logic.get_action('package_show')(context, package_id_dict)
+    if helpers.check_access('package_delete', pkg_dict):
+        thumbnail.remove_thumbnail(context)
+    return package_delete(context, package_id_dict)
+    
+def before_package_update(context, pkg_dict):
+    if helpers.check_access('package_update', pkg_dict):
+        package_name =pkg_dict.get('id') 
+        package = model.Package.get(package_name).as_dict()
+        old_private = package.get('private')
+        new_private = pkg_dict.get('private')
+        if old_private != new_private:
+            old_filename = package.get('extras').get('thumbnail')
+            new_filename = thumbnail.change_filepath(old_filename)
+            pkg_dict.update({'thumbnail': new_filename})
+        return package_update(context, pkg_dict)
\ No newline at end of file

ckanext/odsh/pdf_to_thumbnail/helpers.py

+import thumbnail as tb
+import os
+
+
+def size_of_fmt(num, suffix='B'):
+    for unit in ['','K','M','G','T','P','E','Z']:
+        if abs(num) < 1000.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1000.0
+    return "%.1f%s%s" % (num, 'Y', suffix)
+
+def get_resource_size(resource):
+    resource_size = resource.get('size')
+    if resource_size:
+        return size_of_fmt(resource_size)
+
+def thumbnail_namespace(filename):
+    return "/" + filename
\ No newline at end of file

ckanext/odsh/pdf_to_thumbnail/plugin.py

+import os 
+
+
+#from ckan
+import ckan.plugins as plugins
+from ckan.plugins.toolkit import enqueue_job
+import ckan.plugins.toolkit as toolkit
+
+#pdf_to_thumbnail 
+import thumbnail as tb
+import action as tb_action
+import helpers as tb_helpbers
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class ThumbnailPlugin(plugins.SingletonPlugin):
+    plugins.implements(plugins.IResourceController, inherit=True)
+    plugins.implements(plugins.IConfigurer, inherit=True)
+    plugins.implements(plugins.IActions, inherit=True)
+    plugins.implements(plugins.ITemplateHelpers)
+
+
+#IResourceController
+    def after_create(self, context, resource):        
+        is_PDF_and_size_name = tb.create_thumbnail(context, resource)
+        tb.write_thumbnail_into_package(context, resource, is_PDF_and_size_name)
+        
+    def after_update(self, context, resource):
+        tb.check_and_create_thumbnail_after_update(context, resource)
+                
+    def after_delete(self, context, resources):
+        is_PDF_and_size_name  = tb.create_thumbnail_for_last_resource(context, resources)
+            
+#IConfigurer 
+
+    def update_config(self, config_):
+        storage_path = config_.get('ckan.storage_path')
+        public_dir = os.path.join(storage_path, 'thumbnail')
+        if config_.get('extra_public_paths'):
+            config_['extra_public_paths'] += ',' + public_dir
+        else:
+            config_['extra_public_paths'] = public_dir
+
+#IActions
+
+    def get_actions(self):
+        return {'package_delete': tb_action.before_package_delete,
+                'package_update': tb_action.before_package_update 
+                }
+
+#ITemplateHelpers
+
+    def get_helpers(self):
+        
+        return {
+                'thumbnail_get_resource_size': tb_helpbers.get_resource_size,
+                'thumbnail_namespace':tb_helpbers.thumbnail_namespace
+                }

ckanext/odsh/pdf_to_thumbnail/thumbnail.py

+import os
+import tempfile
+import magic
+import hashlib
+from pdf2image import convert_from_bytes
+import logging
+from ckan.common import config 
+import urllib2
+import requests
+
+import binascii
+import ckan.plugins.toolkit as toolkit
+import ckan.logic as logic
+#from extension
+from ckanext.odsh.lib.uploader import raise_validation_error_if_virus_found
+
+log = logging.getLogger(__name__)
+ 
+
+def get_filename_from_context(context):
+    package = context.get('package')
+    package_id = package.id
+    package= toolkit.get_action('package_show')(context, {'id': package_id})
+    thumbnail = package.get('thumbnail') 
+    return  thumbnail
+
+def get_filepath_for_thumbnail(filename):
+    if filename:
+        return config.get('ckan.storage_path') + "/thumbnail/" + filename
+    return config.get('ckan.storage_path') + "/thumbnail/"
+
+def congante_filename(filename):
+    return filename + ".jpg"
+
+def get_filepath_to_resource(resource):
+    resource_id = resource.get('id')
+    directory = config.get('ckan.storage_path') + '/resources/'
+    path = directory + resource_id[0:3] + '/' + resource_id[3:6] + '/' +  resource_id[6:]
+    return path
+
+def random_filename():
+    number = binascii.b2a_hex(os.urandom(15))
+    filename = 'thumbnail_picture_' + str(number)    
+    full_filename = congante_filename(filename)
+    filepath = get_filepath_for_thumbnail(full_filename)
+    if os.path.exists(filepath):
+        filename = random_filename()
+    return filename
+
+def change_filepath(old_filename):    
+    old_filepath = get_filepath_for_thumbnail(old_filename)
+    new_filename = congante_filename(random_filename())
+    new_filepath = get_filepath_for_thumbnail(new_filename)
+    try:
+        os.renames(old_filepath, new_filepath)
+        return new_filename
+    except OSError:
+        log.warning('The file path "{}"  of package was not found.'.format(old_filepath))
+    
+
+def create_thumbnail_from_file(file, old_filename):
+    width = config.get('ckan.thumbnail.size.width', 300)
+    filename = random_filename()
+    file.seek(0)
+    file_read = file.read()
+    directory = get_filepath_for_thumbnail('')
+    if old_filename:
+        old_filepath = get_filepath_for_thumbnail(congante_filename(old_filename))
+        if os.path.exists(old_filepath):
+            os.remove(old_filepath)
+    convert_from_bytes(file_read,
+                       dpi=50,
+                       output_folder=directory,
+                       output_file=filename,
+                       single_file=True,
+                       first_page=0,
+                       last_page=0,
+                       fmt='jpg'
+                       )
+    return congante_filename(filename)
+
+def create_thumbnail_from_url(resource, old_filename):
+    return 'Not PDF', None, None
+
+    #  Requests does not work on a server but on a local test enviroment.
+    '''
+    resource_url = resource.get('url')
+
+    with requests.Session() as s:
+        r = grequests.head(resource_url, callback=timeme, timeout=3, allow_redirects=False, stream = False, session=s)
+        response2 = grequests.map(r, size=max_workers, exception_handler=handle_errors)
+    
+    request = urllib2.Request(resource_url)
+    response = urllib2.urlopen(request, 'rb') 
+    
+    
+    if response.code == 200:
+        filetowrite = response.read()
+        raise_validation_error_if_virus_found(filetowrite, response.read())
+        file_type = magic.from_buffer(response.read(), mime = True)
+        header = response.headers
+        resource_size = header.get('Content-Length')
+        
+            
+        max_available_memory = config.get('ckan.max_available_memory', 250000000)  #In Bytes ca. 250 MB
+        with tempfile.SpooledTemporaryFile(max_size=max_available_memory) as file:
+            file.write(filetowrite)
+            #file_type = magic.from_buffer(file, mime = True)
+            #if file_type == 'application/pdf':
+            new_filename = create_thumbnail_from_file(file, old_filename)        
+            return 'is PDF', resource_size, new_filename
+            #else:
+            #    return 'Not PDF', resource_size, None
+   ''' 
+def create_thumbnail_from_memory(resource, old_filename):
+    path = get_filepath_to_resource(resource)
+    file_type = magic.from_file(path, mime = True)
+    resource_size = os.path.getsize(path)
+    log.warning('Debugging Output create_thumbnail_from_memory: "{}".'.format(path))
+    if file_type == 'application/pdf':
+        with open(path, 'rb') as file:
+            new_filename = create_thumbnail_from_file(file, old_filename)
+        return 'is PDF', resource_size, new_filename
+    else:
+        return "Not PDF", resource_size, None
+
+def remove_thumbnail(context):
+    old_filename = get_filename_from_context(context)
+    old_filepath = get_filepath_for_thumbnail(old_filename)
+    if os.path.exists(old_filepath):
+            os.remove(old_filepath)
+
+def create_thumbnail(context, resource):
+    old_filename = get_filename_from_context(context)
+    url_type = resource.get('url_type')
+    if resource.get('url_type') == 'upload':
+        is_PDF_and_size_name = create_thumbnail_from_memory(resource, old_filename)
+    else:
+        is_PDF_and_size_name = create_thumbnail_from_url(resource, old_filename)
+    return is_PDF_and_size_name   
+
+def check_and_create_thumbnail_after_update(context, resource):
+    package_id = resource.get('package_id')
+    package = toolkit.get_action('package_show')(context, {'id': package_id})
+    resources = package.get('resources')
+    last_resource = resources.pop()
+    last_resource_id = last_resource.get('id')
+    resource_id = resource.get('id')
+    if last_resource_id == resource_id and resource.get('url_type') != 'upload':
+         is_PDF_and_size_name = create_thumbnail(context, resource)
+         if is_PDF_and_size_name[0]:
+            write_thumbnail_into_package(context, resource, is_PDF_and_size_name)  
+        
+
+def create_thumbnail_for_last_resource(context, resources):
+    if len(resources) > 0:
+        last_resource = resources.pop()
+        is_PDF_and_size_name = create_thumbnail(context, last_resource)
+        if is_PDF_and_size_name[0] == 'Not PDF':
+            is_PDF_and_size_name = create_thumbnail_for_last_resource(context, resources)
+            return is_PDF_and_size_name
+        else:
+            write_thumbnail_into_package(context, last_resource, is_PDF_and_size_name)
+    else:
+        remove_thumbnail(context)
+        package = context.get('package')
+        package_id = package.id
+        package= toolkit.get_action('package_show')(context, {'id': package_id})
+        package.update({'thumbnail': None})
+        toolkit.get_action('package_update')(context, package)
+
+def write_thumbnail_into_package(context, resource, is_PDF_and_size_name):
+        package_id = resource.get('package_id')
+        package = toolkit.get_action('package_show')(context, {'id': package_id})
+        resources = package.get('resources')
+        last_resource = resources.pop()
+        if not last_resource.get('size'):
+            last_resource.update({'size': is_PDF_and_size_name[1]})
+        resources.append(last_resource)
+        package.update({'resources': resources})
+        if is_PDF_and_size_name[2]:
+            package.update({'thumbnail': is_PDF_and_size_name[2]})
+        toolkit.get_action('package_update')(context, package)
\ No newline at end of file

ckanext/odsh/plugin.py

@@ -124,6 +124,11 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 toolkit.get_validator('ignore_missing'),
                 toolkit.get_converter('convert_to_extras')
             ],
+            'thumbnail': [
+                toolkit.get_validator('ignore_missing'),
+                toolkit.get_converter('convert_to_extras')
+            ],
+
         })
         return schema
     
@@ -137,6 +142,10 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 toolkit.get_converter('convert_from_extras'),
                 toolkit.get_validator('ignore_missing')
             ],
+            'thumbnail': [
+                toolkit.get_converter('convert_from_extras'),
+                toolkit.get_validator('ignore_missing')
+            ],
         })
         return schema
 

ckanext/odsh/templates/package/read.html

@@ -97,5 +97,15 @@
     </div>
 {% endfor %}
 </div>
+{% block thumbnail %}
+{% set thumbnail =  pkg.get('thumbnail') %}
+  {% if thumbnail %}
+    {% set picture = h.thumbnail_namespace(thumbnail) %}
+    <div>    
+    <img src= "{{ picture }}" alt= "Vorschau" width="60" height="90" />     
+    </div>  
+   {% endif %}
+{% endblock %}
 
 {% endblock %}
+

ckanext/odsh/templates/package/snippets/resource_item.html

@@ -33,6 +33,12 @@
         {% endif %}
         {% endblock %}
       </div>
+      <div>
+        {% set resource_size = h.thumbnail_get_resource_size(res) %}
+        {% if resource_size %}
+            {{ resource_size }}
+        {% endif %}
+      </div>
       </div>
       {% if(res.format) %}
         <a href="{{ download }}">

ckanext/odsh/templates/snippets/package_item.html

@@ -28,6 +28,7 @@ Example:
 {% set subject_text = h.odsh_extract_value_from_extras(package.extras,'subject_text') if h.odsh_extract_value_from_extras(package.extras,'subject_text') else '-'  %}
 {% set daterange = h.tpsh_get_daterange_prettified(package) %}
 {% set language_of_package = h.tpsh_get_language_of_package(package) %}
+{% set thumbnail = package.get('thumbnail') %}
 
 {% block package_item %}
 <div class="container-fluid odsh-dataset-item">
@@ -84,6 +85,9 @@ Example:
     <p>{{ _('issued') }}: {{issued}} </p>
     {% if language_of_package != 'Deutsch' and language_of_package %}
       <p>{{ _('language') }}:{{ language_of_package }}</p>
+    {% endif %}
+    {% if thumbnail %}
+    <img src= "{{ thumbnail }}" alt= "Vorschau" width="60" height="90" /> 
    {% endif%}
     {% endblock notes %}
   </div>
@@ -93,3 +97,5 @@ Example:
   {% endblock content %}
 </div>
 {% endblock package_item %}
+
+