pdf_to_thumbnail aus Transparenzportal

ckanext/odsh/pdf_to_thumbnail/action.py

@@ -4,8 +4,7 @@ import ckan.lib.helpers as helpers
 from ckan.logic.action.update import package_update
 from ckan.logic.action.delete import package_delete
 
-#from thumbnail
-import thumbnail as thumbnail
+import thumbnail
 
 
 def before_package_delete(context, package_id_dict):
@@ -23,7 +22,7 @@ def before_package_update(context, pkg_dict):
         old_filename = package.get('thumbnail')
         if old_filename:
             if str(old_private) != str(new_private):
-                new_filename = thumbnail.change_filepath(old_filename)
+                new_filename = thumbnail.rename_thumbnail_to_random_name(old_filename)
                 pkg_dict['extras'].append({'key': 'thumbnail', 'value': new_filename})
             elif not pkg_dict.get('thumbnail'): 
                 pkg_dict['extras'].append({'key': 'thumbnail', 'value': old_filename})

ckanext/odsh/pdf_to_thumbnail/plugin.py

 import os 
 
-
 #from ckan
 import ckan.plugins as plugins
 
@@ -22,14 +21,15 @@ class ThumbnailPlugin(plugins.SingletonPlugin):
 
 #IResourceController
     def after_create(self, context, resource):
-        _, filename = thumbnail.create_thumbnail(context, resource)
-        thumbnail.write_thumbnail_into_package(context, resource, filename)
+        resources = thumbnail.resources_of_containing_package(resource)
+        thumbnail.create_thumbnail_if_none_in_package(context, resources)
         
     def after_update(self, context, resource):
-        thumbnail.check_and_create_thumbnail_after_update(context, resource)
+        resources = thumbnail.resources_of_containing_package(resource)
+        thumbnail.create_thumbnail_if_none_in_package(context, resources)
                 
     def after_delete(self, context, resources):
-        thumbnail.create_thumbnail_for_last_resource(context, resources)
+        thumbnail.create_thumbnail_if_none_in_package(context, resources)
             
 #IConfigurer 
 

ckanext/odsh/pdf_to_thumbnail/thumbnail.py

@@ -7,7 +7,7 @@ from ckan.common import config
 import urllib2
 import requests
 
-import binascii
+from binascii import b2a_hex
 import ckan.plugins.toolkit as toolkit
 import ckan.logic as logic
 #from extension
@@ -16,155 +16,212 @@ import ckan.logic as logic
 log = logging.getLogger(__name__)
 
 
-def get_filename_from_context(context):
+def create_thumbnail(context, resource):
+    '''
+    main entry point into this module
+    this function is called from pdf_to_thumbnail.plugin
+    '''
+    old_filename = _get_filename_from_context(context)
+    url_type = resource.get('url_type')
+    if url_type == 'upload':
+        is_PDF, filename = _create_thumbnail_from_memory(resource, old_filename)
+    else:
+        is_PDF, filename = (False, None)
+    return is_PDF, filename  
+
+
+def _get_filename_from_context(context):
     package = context.get('package')
     package_id = package.id
-    package= toolkit.get_action('package_show')(context, {'id': package_id})
+    package= toolkit.get_action('package_show')(None, {'id': package_id})
     thumbnail = package.get('thumbnail') 
     return  thumbnail
 
-def get_filepath_for_thumbnail(filename):
-    if filename:
-        return config.get('ckan.storage_path') + "/thumbnail/" + filename
-    return config.get('ckan.storage_path') + "/thumbnail/"
 
-def concatenate_filename(filename):
-    return filename + ".jpg"
+def _create_thumbnail_from_memory(resource, old_filename):
+    filepath = get_resource_path(resource)
+    is_PDF = _is_pdf(filepath)
+    if is_PDF:
+        with open(filepath, 'rb') as file:
+            new_filename = _create_thumbnail_from_file(file)
+        if old_filename:
+            ThumbnailPath.from_filename(old_filename).remove()
+        return is_PDF, new_filename
+    else:
+        return is_PDF, None
+
 
-def get_filepath_to_resource(resource):
+def get_resource_path(resource):
+    # see https://stackoverflow.com/questions/46572402/where-does-ckan-store-the-files-pushed-to-datastore-filestore
     resource_id = resource.get('id')
-    directory = config.get('ckan.storage_path') + '/resources/'
-    #looked up how resources are saved, by locating the keyword resources in the OS 
-    path = directory + resource_id[0:3] + '/' + resource_id[3:6] + '/' +  resource_id[6:]
-    return path
+    filepath = os.path.join(
+        config.get('ckan.storage_path'),
+        'resources',
+        resource_id[0:3],
+        resource_id[3:6],
+        resource_id[6:]
+    )
+    return filepath
 
-def random_filename():
-    number = binascii.b2a_hex(os.urandom(15))
-    filename = 'thumbnail_picture_' + str(number)    
-    full_filename = concatenate_filename(filename)
-    filepath = get_filepath_for_thumbnail(full_filename)
-    if os.path.exists(filepath):
-        filename = random_filename()
-    return filename
-
-def change_filepath(old_filename):    
-    old_filepath = get_filepath_for_thumbnail(old_filename)
-    new_filename = concatenate_filename(random_filename())
-    new_filepath = get_filepath_for_thumbnail(new_filename)
-    try:
-        os.renames(old_filepath, new_filepath)
-        return new_filename
-    except OSError:
-        log.warning('The file path "{}"  of package was not found.'.format(old_filepath))
+
+def _is_pdf(filepath):
+    file_type = magic.from_file(filepath, mime = True)
+    return file_type == 'application/pdf'
 
 
-def create_thumbnail_from_file(file, old_filename):
+def _create_thumbnail_from_file(file):
     width = config.get('ckan.thumbnail.size.width', 410)
-    filename = random_filename()
+    new_thumbnail = ThumbnailPath.from_unique_random_name()
     file.seek(0)
     file_read = file.read()
-    directory = get_filepath_for_thumbnail('')
-    if old_filename:
-        old_filepath = get_filepath_for_thumbnail(concatenate_filename(old_filename))
-        if os.path.exists(old_filepath):
-            os.remove(old_filepath)
-    convert_from_bytes(file_read,
+    convert_from_bytes(
+        file_read,
         size=(width, None),
-                       output_folder=directory,
-                       output_file=filename,
+        output_folder=new_thumbnail.folder,
+        output_file=new_thumbnail.filename,
         single_file=True,
         first_page=0,
         last_page=0,
         fmt='jpg'
     )
-    return concatenate_filename(filename)
+    return new_thumbnail.filename_with_extension
 
 
-def create_thumbnail_from_url(resource, old_filename):
-    resource_url = resource.get('url')
-    request = urllib2.Request(resource_url)
-    response = urllib2.urlopen(request, timeout = 100000) 
-    
-    
-    if response.code == 200:
-        filetowrite = response.read()
-        # function is set to private in ckanext.odsh.lib.uploader
-        # raise_validation_error_if_virus_found(filetowrite, response.read())
-        file_type = magic.from_buffer(response.read(), mime = True)
-        header = response.headers
-        resource_size = header.get('Content-Length')
-        
+def thumbnail_folder():
+    return os.path.join(
+        config.get('ckan.storage_path'),
+        'thumbnail',
+    )
 
-        max_available_memory = config.get('ckan.max_available_memory', 250000000)  #In Bytes ca. 250 MB
-        with tempfile.SpooledTemporaryFile(max_size=max_available_memory) as file:
-            file.write(filetowrite)
 
-            new_filename = create_thumbnail_from_file(file, old_filename)        
-            return True, new_filename
+def rename_thumbnail_to_random_name(old_filename):
+    '''
+    used by pdf_to_thumbnail.action
+    '''
+    old_filepath = ThumbnailPath.from_filename_with_extension(old_filename)
+    new_filepath = ThumbnailPath.from_unique_random_name()
+    try:
+        os.renames(old_filepath.full_filename, new_filepath.full_filename)
+        return new_filepath.filename_with_extension
+    except OSError:
+        log.warning('The file path "{}"  of package was not found.'.format(old_filepath))
      
-def create_thumbnail_from_memory(resource, old_filename):
-    path = get_filepath_to_resource(resource)
-    file_type = magic.from_file(path, mime = True)
-    if file_type == 'application/pdf':
-        with open(path, 'rb') as file:
-            new_filename = create_thumbnail_from_file(file, old_filename)
-        is_PDF = True
-        return is_PDF, new_filename
-    else:
-        is_PDF = False
-        return is_PDF,  None
 
 def remove_thumbnail(context):
-    old_filename = get_filename_from_context(context)
+    '''
+    used by pdf_to_thumbnail.action
+    '''
+    old_filename = _get_filename_from_context(context)
     if old_filename:
-        old_filepath = get_filepath_for_thumbnail(old_filename)
-        if os.path.exists(old_filepath):
-            os.remove(old_filepath)
+        ThumbnailPath.from_filename_with_extension(old_filename).remove()
 
-def create_thumbnail(context, resource):
-    log.debug('create_thumbnail')
-    old_filename = get_filename_from_context(context)
-    url_type = resource.get('url_type')
-    if url_type == 'upload':
-        is_PDF,  filename = create_thumbnail_from_memory(resource, old_filename)
-    else:
-        is_PDF,  filename = create_thumbnail_from_url(resource, old_filename)
-    return is_PDF,  filename   
 
-def check_and_create_thumbnail_after_update(context, resource):
-    log.debug('check_and_create_thumbnail_after_update')
+def resources_of_containing_package(resource):
+    #todo: change arg order
+    '''
+    used by pdf_to_thumbnail.plugin
+    '''
     package_id = resource.get('package_id')
-    package = toolkit.get_action('package_show')(context, {'id': package_id})
+    package = toolkit.get_action('package_show')(None, {'id': package_id})
     resources = package.get('resources')
-    if len(resources) > 0:
-        last_resource = resources.pop()
-        last_resource_id = last_resource.get('id')
-        resource_id = resource.get('id')
-    if last_resource_id == resource_id and resource.get('url_type') != 'upload':
+    return resources
+        
+
+def create_thumbnail_if_none_in_package(context, resources):
+    '''
+    used by pdf_to_thumbnail.plugin
+    loops through a package's resources in the order they have been uploaded
+    and for each tries to create a thumbnail until it succeeds.
+    If the package already has a thumbnail the creation step is skipped
+    '''
+    package_dict = _get_package_dict_from_context(context)
+    if not _has_thumbnail(package_dict):
+        any(_try_create_thumbnail(context, r) for r in resources)
+
+
+def _get_package_dict_from_context(context):
+    package_id = context.get('package').id
+    package_dict = toolkit.get_action('package_show')(None, {'id': package_id})
+    return package_dict
+
+
+def _has_thumbnail(package_dict):
+    thumbnail = package_dict.get('thumbnail')
+    return bool(thumbnail)
+
+
+def _try_create_thumbnail(context, resource):
     is_PDF, filename = create_thumbnail(context, resource)
-        if is_PDF:
-            write_thumbnail_into_package(context, resource, filename)  
+    success = is_PDF
+    if success:
+        _write_thumbnail_into_package(context, filename)
+    return success
 
 
-def create_thumbnail_for_last_resource(context, resources):
-    if len(resources) > 0:
-        last_resource = resources.pop()
-        is_PDF, filename = create_thumbnail(context, last_resource)
-        if not is_PDF:
-            create_thumbnail_for_last_resource(context, resources)
-        else:
-            write_thumbnail_into_package(context, last_resource, filename)
+def _write_thumbnail_into_package(context, filename):
+    package_dict = _get_package_dict_from_context(context)
+    if filename:
+        package_dict.update({'thumbnail': filename})
+    toolkit.get_action('package_update')(None, package_dict)
+    
+
+class ThumbnailPath(object):
+    '''
+    utility class to manage the path of thumbnail pictures
+    '''
+
+    def __init__(self, folder, filename, extension):
+        self.folder = folder
+        self.filename = filename
+        self.extension = extension
+    
+    _EXTENSION = '.jpg'
+    
+    @staticmethod
+    def from_filename(filename):
+        '''
+        filename without extension (i.e. '.jpg')
+        '''
+        return ThumbnailPath(thumbnail_folder(), filename, ThumbnailPath._EXTENSION)
+    
+    @staticmethod
+    def from_filename_with_extension(filename_with_extension):
+        '''
+        limited to one dot in filename
+        '''
+        tokens = filename_with_extension.split('.')
+        if len(tokens) == 1:
+            filename = filename_with_extension
+            extension = ''
         else:
-        remove_thumbnail(context)
-        package = context.get('package')
-        package_id = package.id
-        package= toolkit.get_action('package_show')(context, {'id': package_id})
-        package.update({'thumbnail': None})
-        toolkit.get_action('package_update')(context, package)
+            filename = '.'.join(tokens[:-1])
+            extension = '.'.join(['', tokens[-1]])
+        return ThumbnailPath(thumbnail_folder(), filename, extension)
+
+    @staticmethod
+    def from_unique_random_name():
+        thumbnail_path = ThumbnailPath._from_random_name()
+        if thumbnail_path.exists():
+            return ThumbnailPath.from_unique_random_name()
+        return thumbnail_path
+    
+    @staticmethod
+    def _from_random_name():
+        number = b2a_hex(os.urandom(15))
+        filename = 'thumbnail_picture_' + str(number)
+        return ThumbnailPath.from_filename(filename)
     
-def write_thumbnail_into_package(context, resource, filename):
-        package_id = resource.get('package_id')
-        package = toolkit.get_action('package_show')(context, {'id': package_id})
-        if filename:
-            package.update({'thumbnail': filename})
-        toolkit.get_action('package_update')(context, package)
+    @property
+    def filename_with_extension(self):
+        return self.filename + self.extension
+    
+    @property
+    def full_filename(self):
+        return os.path.join(self.folder, self.filename_with_extension)
+    
+    def exists(self):
+        return os.path.exists(self.full_filename)
+    
+    def remove(self):
+        if os.path.exists(self.full_filename):
+            os.remove(self.full_filename)