Merge branch 'master' of ssh://ssh.bitbucket.dataport.de:7999/odsh/ckanext-odsh

ckanext/odsh/lib/odsh_icap_client.py

 import socket
 import sys
 import time
-import ConfigParser
+from ckan.common import config
+import logging
+
+log = logging.getLogger(__name__)
 
 
 class ODSHICAPRequest(object):
 
-    def __init__(self, FILENAME, FILEBUFF, cfg_file='odsh_icap_client.cfg'):
-        config = ConfigParser.ConfigParser()
-        config.read(cfg_file)
-        self.HOST = '10.61.127.77'
-        self.PORT = 1344
+    def __init__(self, FILENAME, FILEBUFF):
+        self.HOST = config.get("ckanext.odsh.icap.host", "10.61.127.77")   #'10.61.127.77'
+        self.PORT = config.get("ckanext.odsh.icap.port", 1344)
+
         self.CLIENTIP = '127.0.0.1'
         self.FILENAME = FILENAME
         self.FILEBUFF = FILEBUFF
 
     def send(self):
-        print("----- Starting ICAP-Request via RESPMOD -----")
+        log.info("----- Starting ICAP-Request via RESPMOD -----")
 
         # socket connect
         try:
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         except socket.error as msg:
-            sys.stderr.write("[ERROR] %s\n" % msg[1])
+            log.error(msg[1])
             sys.exit(1)
 
         try:
             sock.connect((self.HOST, self.PORT))
         except socket.error as msg:
-            sys.stderr.write("[ERROR] %s\n" % msg[1])
+            log.error(msg[1])
             sys.exit(2)
 
         # create and send header
@@ -43,7 +45,7 @@ class ODSHICAPRequest(object):
         data_response = self._recvall(sock)
         response_object = self._parse_response(data_response)
 
-        print("----- Finished ICAP-Request via RESPMOD -----")
+        log.info("----- Finished ICAP-Request via RESPMOD -----")
 
         return response_object
 
@@ -71,12 +73,12 @@ class ODSHICAPRequest(object):
         return icapRequest
 
     def _sendfile(self, fileBuffer, sock):
-        print('start sending file')
+        log.info('Start sending file.')
         PACK_SIZE = 1024 # in bytes
 
         l = fileBuffer.read(PACK_SIZE)
         while(l):
-            print('sending %d bytes of data...' % len(l))
+            log.info('Sending {} bytes of data...'.format(len(l)))
             sock.send('{:02X}'.format(len(l)).encode())
             sock.send("\r\n".encode())
             sock.send(l)
@@ -84,22 +86,22 @@ class ODSHICAPRequest(object):
             l = fileBuffer.read(PACK_SIZE)
 
     def _sendfile_old(self, fileName, sock):
-        print('start sending file')
+        log.info('OLD: Start sending file.')
         PACK_SIZE = 1024 # in bytes
     
         with open(fileName) as f:
             l = f.read(PACK_SIZE)
             while(l):
-                print('sending %d bytes of data...' % len(l))
+                log.info('Sending {} bytes of data...'.format(len(l)))
                 sock.send('{:02X}'.format(len(l)).encode())
                 sock.send("\r\n".encode())
                 sock.send(l)
                 sock.send("\r\n".encode())
                 l = f.read(PACK_SIZE)
-        print('done sending')
+        log.info('Done sending.')
     
     def _recvall(self, sock):
-        print('receiving response from icap server')
+        log.info('Receiving response from icap server...')
         BUFF_SIZE = 4096 # 4 KiB
         data = b''
         while True:
@@ -111,7 +113,7 @@ class ODSHICAPRequest(object):
         return data
 
     def _parse_response(self, data_response):
-        print('parsing response')
+        log.info('Parsing response...')
         lines = data_response.split('\r\n')
         http_status_code = self._parse_response_http_statuscode(lines)
         http_block = self._parse_block(lines, 'HTTP/1.1')
@@ -160,7 +162,9 @@ class ODSHParsedICAPResponse(object):
 
     def virus_found(self):
         if (self.http_status_code != 200) and (self.http_status_code != 403):
-            raise UnknownResponseException('Received an unknown http response code: %d' % self.http_status_code)
+            msg = 'Received an unknown http response code: {}'.format(self.http_status_code)
+            log.warning(msg)
+            raise UnknownResponseException(msg)
         return self.http_status_code != 200
 
 

ckanext/odsh/lib/uploader.py

@@ -10,8 +10,10 @@ log = logging.getLogger(__name__)
 class ODSHResourceUpload(ResourceUpload):
 
     def __init__(self, resource):
+        log.info("Resource({}) uploaded.".format(resource))
         super(ODSHResourceUpload, self).__init__(resource)
         if self._icap_virus_found():
+            log.info("Found Virus!")
             raise logic.ValidationError({'upload': ['Virus gefunden']})
         
     def _icap_virus_found(self):

ckanext/odsh/plugin.py

@@ -564,18 +564,32 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         return pkg_dict
     
     def _is_package_new(self, pkg_dict):
-        date_package_created = self._get_date_from_string(pkg_dict['metadata_created'])
+        date_package_created_as_str = self._get_date_of_package_creation_from_pkg_dict(pkg_dict)
+        if date_package_created_as_str == None:
+            is_new = False
+        else:
+            date_package_created = self._get_date_from_string(date_package_created_as_str)
+            if date_package_created == None:
+                is_new = False
+            else:
                 is_new = odsh_helpers.is_within_last_month(date_package_created)
         return is_new
     
+    def _get_date_of_package_creation_from_pkg_dict(self, pkg_dict):
+        if 'extras' in pkg_dict:
+            extras = pkg_dict['extras']
+            issued = odsh_helpers.odsh_extract_value_from_extras(extras=extras, key='issued') # is None if issued not in extras
+            return issued
+        else:
+            return None
+    
     def _get_date_from_string(self, date_time_str):
-        # todo: update this function if used in different context
-        date_time_format = '%Y-%m-%dT%H:%M:%S.%f' #e.g. u'2019-06-12T11:56:25.059563'
+        date_time_format = '%Y-%m-%dT%H:%M:%S' #e.g. u'2019-06-12T11:56:25'
         try:
             date_time = datetime.datetime.strptime(date_time_str, date_time_format)
-        except ValueError:
-            # if date cannot be converted from string fall back to 1.1.2000
-            date = datetime.date(2000, 1, 1)
             date = date_time.date()
+        except ValueError:
+            # if date cannot be converted from string return None
+            date = None
         return date
 

ckanext/odsh/tests/icap_sever_mock.py

+import socket
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import time
+import requests
+import pdb
+import threading
+import asyncore
+
+
+hostName = "localhost"
+hostPort = 5002
+
+# data = "test"
+# with open('catalog.rdf', 'r') as myfile:
+#         data=myfile.read()
+
+# TODO: better was to set data on RequestHandler
+data = ""
+
+class RequestHandler(BaseHTTPRequestHandler):
+
+    # GET
+    def do_GET(self):
+        self.send_response(requests.codes.ok)
+        # self.send_header('Content-Type', 'application/json; charset=utf-8')
+        self.send_header(
+            'Content-Type', 'application/rdf+xml; charset=utf-8')
+        self.end_headers()
+        self.wfile.write(data.encode("utf-8"))
+
+    # POST
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length'])
+        post_data = self.rfile.read(content_length)
+        self.send_response(200)
+        print(post_data)
+        self.wfile.write("".encode("utf-8"))  # send back to client
+
+    def do_HEAD(self):
+        self.send_response(requests.codes.ok)
+        self.send_header('Content-Type', 'application/json; charset=utf-8')
+        self.end_headers()
+
+
+class HarvestServerMock(threading.Thread):
+    def __init__(self):
+        # init thread
+        self._stop_event = threading.Event()
+        self.thread_name = self.__class__
+        self.server = HTTPServer((hostName, hostPort), RequestHandler)
+        threading.Thread.__init__(
+            self, name=self.thread_name, target=self.server.serve_forever)
+        self.setDaemon(True)
+
+
+#     def run(self):
+#         while not self._stop_event.isSet():
+#             asyncore.loop(timeout=0.01, count=1)
+
+        # pdb.set_trace()
+
+        # try:
+        # except KeyboardInterrupt:
+        #         pass
+
+    def close(self):
+        self.server.server_close()
+       # print(time.asctime(), "Server Stops - %s:%s" % (hostName, hostPort))