From 81c20a5635f8d80bc1531104e82116a7818d8214 Mon Sep 17 00:00:00 2001
From: Thorge Petersen <petersen@rz.uni-kiel.de>
Date: Mon, 5 Jun 2023 10:01:24 +0200
Subject: [PATCH] Removed icap code
---
ckanext/odsh/lib/odsh_icap_client.py | 205 ---------------------------
ckanext/odsh/lib/uploader.py | 23 ---
ckanext/odsh/plugin_odsh_icap.py | 11 --
setup.py | 1 -
4 files changed, 240 deletions(-)
delete mode 100644 ckanext/odsh/lib/odsh_icap_client.py
delete mode 100644 ckanext/odsh/plugin_odsh_icap.py
diff --git a/ckanext/odsh/lib/odsh_icap_client.py b/ckanext/odsh/lib/odsh_icap_client.py
deleted file mode 100644
index fac8b274..00000000
--- a/ckanext/odsh/lib/odsh_icap_client.py
+++ /dev/null
@@ -1,205 +0,0 @@
-import socket
-import sys
-import time
-import logging
-from ckan.common import config
-import ckan.plugins.toolkit as toolkit
-
-log = logging.getLogger(__name__)
-
-def _read_from_config(key):
- value = config.get(key, None)
- if value is None:
- _raise_KeyError_if_not_in_config(key)
- return value
-
-def _raise_KeyError_if_not_in_config(key):
- raise KeyError('key {} is not defined in ckan config file.'.format(key))
-
-
-class ODSHICAPRequest(object):
-
- def __init__(self, FILENAME, FILEBUFF):
- try:
- self.HOST = _read_from_config('ckanext.odsh.icap.host')
- self.PORT = toolkit.asint(_read_from_config('ckanext.odsh.icap.port'))
- self.CLIENTIP = _read_from_config('ckanext.odsh.icap.clientip')
- except KeyError as e:
- log.error(e)
- self.FILENAME = FILENAME
- self.FILEBUFF = FILEBUFF
-
- def send(self):
- print("----- Starting ICAP-Request via RESPMOD -----")
-
- # socket connect
- try:
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- except socket.error as msg:
- sys.stderr.write("[ERROR] %s\n" % msg[1])
- sys.exit(1)
-
- try:
- sock.connect((self.HOST, self.PORT))
- except socket.error as msg:
- sys.stderr.write("[ERROR] %s\n" % msg[1])
- sys.exit(2)
-
- # create and send header
- header = self._get_icap_header(self.FILENAME, self.HOST, self.PORT, self.CLIENTIP).encode()
- sock.send(header)
-
- # send file and terminating signal
- self._sendfile(self.FILEBUFF, sock)
- sock.send('0\r\n\r\n')
-
- # fetch and parse the response
- data_response = self._recvall(sock)
- response_object = self._parse_response(data_response)
-
- print("----- Finished ICAP-Request via RESPMOD -----")
-
- return response_object
-
- def _get_icap_header(self, fileName, host, port, clientIP):
- uniqueInt = time.time() # used to generate "unique" int for disabling cache
-
- icapRequest = 'RESPMOD' + ' ' + 'icap://' + host + ':' + str(port) + '/RESPMOD' + \
- ' ICAP/1.0\r\n' + 'Host: ' + host + ':' + str(port) + '\r\n'
- icapRequest += 'Allow: 204\r\n'
- icapRequest += 'X-Client-IP: ' + clientIP + '\r\n'
-
- httpRequest = "GET http://" + clientIP + "/" + str(uniqueInt).replace('.', '_') + "/" + \
- fileName + ' HTTP/1.1\r\nHost: ' + clientIP + '\r\n\r\n'
-
- httpResponse = 'HTTP/1.1 200 OK\r\n'
- httpResponse += 'Transfer-Encoding: chunked\r\n'
- httpResponse += '\r\n'
-
- httpRequestLength = len(httpRequest)
- httpResponseLength = len(httpResponse)
-
- icapRequest += 'Encapsulated: req-hdr=0, res-hdr=' + str(httpRequestLength) + ', res-body=' + \
- str(httpRequestLength + httpResponseLength) + '\r\n\r\n' + httpRequest + httpResponse;
-
- return icapRequest
-
- def _sendfile(self, fileBuffer, sock):
- print('start sending file')
- PACK_SIZE = 1024 # in bytes
-
- l = fileBuffer.read(PACK_SIZE)
- while(l):
- print(('sending %d bytes of data...' % len(l)))
- sock.send('{:02X}'.format(len(l)).encode())
- sock.send("\r\n".encode())
- sock.send(l)
- sock.send("\r\n".encode())
- l = fileBuffer.read(PACK_SIZE)
-
- def _recvall(self, sock):
- print('receiving response from icap server')
- BUFF_SIZE = 4096 # 4 KiB
- data = b''
- while True:
- part = sock.recv(BUFF_SIZE)
- data += part
- if len(part) < BUFF_SIZE:
- # either 0 or end of data
- break
- return data
-
- def _parse_response(self, data_response):
- print('parsing response')
- lines = data_response.split('\r\n')
- http_status_code = self._parse_response_http_statuscode(lines)
- http_block = self._parse_block(lines, 'HTTP/1.1')
- icap_block = self._parse_block(lines, 'ICAP/1.0')
-
- response_object = ODSHParsedICAPResponse(data_response, http_status_code, http_block, icap_block)
- return response_object
-
- def _parse_response_http_statuscode(self, data_response_lines):
- http_status_code_found = False
- http_status_code = None
- for line in data_response_lines:
- if line.startswith('HTTP/1.1'):
- http_status_code = int(line.split(' ')[1]) # example: HTTP/1.1 403 VirusFound
- http_status_code_found = True
-
- if not http_status_code_found:
- http_status_code = 200 # if no virus is found, no http_status_code is given, defaulting to 200 OK
-
- return http_status_code
-
- def _parse_block(self, data_response_lines, block_start_signal):
- block_data = None
- in_block = False
-
- for line in data_response_lines:
- if line.startswith(block_start_signal):
- in_block = True
- block_data = ''
- if in_block and not len(line):
- in_block = False
- break
- if in_block:
- block_data += line + '\r\n'
-
- return block_data
-
-
-class ODSHParsedICAPResponse(object):
-
- def __init__(self, full_response, http_status_code, http_block, icap_block):
- self.full_response = full_response
- self.http_status_code = http_status_code
- self.http_block = http_block
- self.icap_block = icap_block
-
- def virus_found(self):
- if (self.http_status_code != 200) and (self.http_status_code != 403):
- raise UnknownResponseException('Received an unknown http response code: %d' % self.http_status_code)
- return self.http_status_code != 200
-
-
-class UnknownResponseException(Exception):
- pass
-
-
-def example_print_response(response_object):
- print('')
- print('Example output of response_object:')
- print('')
-
- #print('Full ICAP-Response: ')
- #print(response_object.full_response)
- #print('')
-
- print('HTTP-Status-Code (explicit or implied):')
- print((response_object.http_status_code))
- print('')
-
- print('HTTP-Block:')
- print((response_object.http_block))
- print('')
-
- print('ICAP-Block:')
- print((response_object.icap_block))
- print('')
-
- print('Virus found?')
- print((response_object.virus_found()))
- print('')
-
-
-if __name__ == "__main__":
-
- # example file with virus
- FILENAME = 'test_files/eicar.txt'
-
- # example file without virus
- #FILENAME = 'test_files/lorem-ipsum.pdf'
-
- odsh_parsed_icap_response = ODSHICAPRequest(FILENAME).send()
- example_print_response(odsh_parsed_icap_response)
diff --git a/ckanext/odsh/lib/uploader.py b/ckanext/odsh/lib/uploader.py
index 144a64ae..3ef936af 100644
--- a/ckanext/odsh/lib/uploader.py
+++ b/ckanext/odsh/lib/uploader.py
@@ -3,31 +3,11 @@ from ckan.lib.uploader import ResourceUpload, Upload
import ckan.plugins.toolkit as toolkit
import ckan.plugins.toolkit as tk
-from .odsh_icap_client import ODSHICAPRequest
import logging
import hashlib
log = logging.getLogger(__name__)
-
-def _icap_virus_found(filename, upload_file):
- # the flag skip_icap_virus_check in can be used during development
- skip_icap_virus_check = toolkit.asbool(
- tk.config.get('ckanext.odsh.skip_icap_virus_check', 'False')
- )
- if skip_icap_virus_check:
- log.debug("WARNING: icap virus check skipped, remove parameter ckanext.odsh.skip_icap_virus_check from ckan's ini file")
- return False
- if filename and upload_file:
- response_object = ODSHICAPRequest(filename, upload_file).send()
- return response_object.virus_found()
-
-
-def _raise_validation_error_if_virus_found(filename, upload_file):
- if _icap_virus_found(filename, upload_file):
- raise logic.ValidationError({'upload': ['Virus gefunden']})
-
-
def calculate_hash(upload_file):
upload_file.seek(0)
hash_md5 = hashlib.md5()
@@ -53,7 +33,6 @@ class ODSHResourceUpload(ResourceUpload):
log.debug("Resource({}) uploaded.".format(resource))
super(ODSHResourceUpload, self).__init__(resource)
if hasattr(self, 'filename') and hasattr(self, 'upload_file'):
- _raise_validation_error_if_virus_found(self.filename, self.upload_file)
_raise_validation_error_if_hash_values_differ(self.upload_file, resource)
@@ -72,6 +51,4 @@ class ODSHUpload(Upload):
super(ODSHUpload, self).update_data_dict(data_dict, url_field, file_field, clear_field)
def upload(self, max_size=2):
- if hasattr(self, 'filename') and hasattr(self, 'upload_file'):
- _raise_validation_error_if_virus_found(self.filename, self.upload_file)
super(ODSHUpload, self).upload(max_size)
diff --git a/ckanext/odsh/plugin_odsh_icap.py b/ckanext/odsh/plugin_odsh_icap.py
deleted file mode 100644
index d6462db4..00000000
--- a/ckanext/odsh/plugin_odsh_icap.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import ckan.plugins as plugins
-from ckanext.odsh.lib.uploader import ODSHResourceUpload, ODSHUpload
-
-class OdshIcapPlugin(plugins.SingletonPlugin):
- plugins.implements(plugins.IUploader, inherit=True)
-
- def get_resource_uploader(self, data_dict):
- return ODSHResourceUpload(data_dict)
-
- def get_uploader(self, upload_to, old_filename):
- return ODSHUpload(upload_to, old_filename)
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 99bcc0ff..3fc6e689 100755
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,6 @@ setup(
entry_points='''
[ckan.plugins]
odsh=ckanext.odsh.plugin:OdshPlugin
- odsh_icap=ckanext.odsh.plugin_odsh_icap:OdshIcapPlugin
statistikamtnord_harvester=ckanext.odsh.harvesters:StatistikamtNordHarvester
kiel_harvester=ckanext.odsh.harvesters:KielHarvester
odsh_autocomplete=ckanext.odsh.plugin_odsh_autocomplete:OdshAutocompletePlugin
--
GitLab