Skip to content
Snippets Groups Projects
Commit ca79a8b3 authored by chbaeh's avatar chbaeh
Browse files

fix value extraction from extras

parent 5ec249ab
No related branches found
No related tags found
No related merge requests found
...@@ -15,6 +15,7 @@ import hashlib ...@@ -15,6 +15,7 @@ import hashlib
get_action = logic.get_action get_action = logic.get_action
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def odsh_openness_score_dataset_html(dataset): def odsh_openness_score_dataset_html(dataset):
score = 0 score = 0
#dataset = json.loads(dataset) #dataset = json.loads(dataset)
...@@ -26,7 +27,8 @@ def odsh_openness_score_dataset_html(dataset): ...@@ -26,7 +27,8 @@ def odsh_openness_score_dataset_html(dataset):
if r_qa: if r_qa:
try: try:
qa = None qa = None
if isinstance(r_qa, basestring): # r_qa might be a string of a dictionary when 'dataset' is send from solr # r_qa might be a string of a dictionary when 'dataset' is send from solr
if isinstance(r_qa, basestring):
qa = ast.literal_eval(r_qa) qa = ast.literal_eval(r_qa)
else: else:
qa = r_qa qa = r_qa
...@@ -38,6 +40,7 @@ def odsh_openness_score_dataset_html(dataset): ...@@ -38,6 +40,7 @@ def odsh_openness_score_dataset_html(dataset):
e.__class__.__name__, unicode(e), traceback.format_exc()) e.__class__.__name__, unicode(e), traceback.format_exc())
return score return score
def odsh_get_resource_details(resource_id): def odsh_get_resource_details(resource_id):
resource_details = toolkit.get_action('resource_show')( resource_details = toolkit.get_action('resource_show')(
data_dict={'id': resource_id}) data_dict={'id': resource_id})
...@@ -52,6 +55,7 @@ def odsh_get_resource_views(pkg_dict, resource): ...@@ -52,6 +55,7 @@ def odsh_get_resource_views(pkg_dict, resource):
return get_action('resource_view_list')( return get_action('resource_view_list')(
context, {'id': resource['id']}) context, {'id': resource['id']})
def odsh_get_bounding_box(pkg_dict): def odsh_get_bounding_box(pkg_dict):
try: try:
extras = pkg_dict.get('extras') extras = pkg_dict.get('extras')
...@@ -71,13 +75,14 @@ def odsh_get_bounding_box(pkg_dict): ...@@ -71,13 +75,14 @@ def odsh_get_bounding_box(pkg_dict):
e.__class__.__name__, unicode(e), traceback.format_exc()) e.__class__.__name__, unicode(e), traceback.format_exc())
return None return None
def compute_bounding_box(coords): def compute_bounding_box(coords):
if len(coords) == 0: if len(coords) == 0:
return None return None
coords = [c for sublist in coords for c in sublist] coords = [c for sublist in coords for c in sublist]
if type(coords[0][0]) == list: if type(coords[0][0]) == list:
## multipolygon # multipolygon
coords = [c for sublist in coords for c in sublist] coords = [c for sublist in coords for c in sublist]
minx = min(coords, key=lambda t: t[0])[0] minx = min(coords, key=lambda t: t[0])[0]
...@@ -87,6 +92,7 @@ def compute_bounding_box(coords): ...@@ -87,6 +92,7 @@ def compute_bounding_box(coords):
return [maxx, minx, maxy, miny] return [maxx, minx, maxy, miny]
def odsh_get_spatial_text(pkg_dict): def odsh_get_spatial_text(pkg_dict):
extras = pkg_dict.get('extras') extras = pkg_dict.get('extras')
spatial = None spatial = None
...@@ -98,22 +104,27 @@ def odsh_get_spatial_text(pkg_dict): ...@@ -98,22 +104,27 @@ def odsh_get_spatial_text(pkg_dict):
return spatial return spatial
return None return None
def odsh_render_datetime(datetime_, date_format='{0.day:02d}.{0.month:02d}.{0.year:4d}'): def odsh_render_datetime(datetime_, date_format='{0.day:02d}.{0.month:02d}.{0.year:4d}'):
dt = parser.parse(datetime_) dt = parser.parse(datetime_)
return date_format.format(dt) return date_format.format(dt)
def odsh_upload_known_formats(): def odsh_upload_known_formats():
value = config.get('ckanext.odsh.upload_formats', []) value = config.get('ckanext.odsh.upload_formats', [])
value = toolkit.aslist(value) value = toolkit.aslist(value)
return value return value
def odsh_encodeurl(url): def odsh_encodeurl(url):
return urllib.quote(url, safe='') return urllib.quote(url, safe='')
def odsh_create_checksum(in_string): def odsh_create_checksum(in_string):
hashstring = hashlib.md5(in_string.encode('utf-8')).hexdigest() hashstring = hashlib.md5(in_string.encode('utf-8')).hexdigest()
return int(hashstring, base=16) return int(hashstring, base=16)
def odsh_extract_error(key, errors): def odsh_extract_error(key, errors):
if not errors or not ('extras' in errors): if not errors or not ('extras' in errors):
return None return None
...@@ -124,9 +135,12 @@ def odsh_extract_error(key, errors): ...@@ -124,9 +135,12 @@ def odsh_extract_error(key, errors):
if error.startswith(key): if error.startswith(key):
return error.replace(key+':', '') return error.replace(key+':', '')
def odsh_extract_value_from_extras(extras, key): def odsh_extract_value_from_extras(extras, key):
if not extras: if not extras:
return None return None
for item in extras: for item in extras:
if 'key' in item and item['key'].lower() == key.lower(): if 'key' in item and item['key'].lower() == key.lower():
if 'value' in item:
return item['value'] return item['value']
return None
...@@ -132,7 +132,6 @@ def odsh_validate_extra_date(key, field, data, errors, context): ...@@ -132,7 +132,6 @@ def odsh_validate_extra_date(key, field, data, errors, context):
# Statistikamt Nord does not always provide temporal_start/end, # Statistikamt Nord does not always provide temporal_start/end,
# but their datasets have to be accepted as they are. # but their datasets have to be accepted as they are.
if not ('id',) in data or data[('id',)][:7] != 'StaNord': if not ('id',) in data or data[('id',)][:7] != 'StaNord':
print(field+' raise missing')
raise toolkit.Invalid(field+':odsh_'+field+'_error_label') raise toolkit.Invalid(field+':odsh_'+field+'_error_label')
else: else:
try: try:
...@@ -140,9 +139,7 @@ def odsh_validate_extra_date(key, field, data, errors, context): ...@@ -140,9 +139,7 @@ def odsh_validate_extra_date(key, field, data, errors, context):
# no matter if the date is like "2012-01-01" or "2012-01-01T00:00:00" # no matter if the date is like "2012-01-01" or "2012-01-01T00:00:00"
datetime.datetime.strptime(value.split('T')[0],'%Y-%m-%d').isoformat() datetime.datetime.strptime(value.split('T')[0],'%Y-%m-%d').isoformat()
except ValueError: except ValueError:
print(field+' raise wrong')
raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label') raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label')
print(field+' raise nothing')
def odsh_validate_extra_date_factory(field): def odsh_validate_extra_date_factory(field):
......
import ckan.tests.helpers as helpers import contextlib
import errno
import functools
from ckan.common import config
def odsh_test(): return helpers.change_config('ckanext.odsh.spatial.mapping',
'file:///usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/tests/spatial_mapping.csv') def odsh_test():
\ No newline at end of file return change_config()
def change_config():
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
with changed_config():
return func(*args, **kwargs)
return wrapper
return decorator
@contextlib.contextmanager
def changed_config():
_original_config = config.copy()
config['ckanext.odsh.spatial.mapping'] = 'file:///usr/lib/ckan/default/src/ckanext-odsh/ckanext/odsh/tests/spatial_mapping.csv'
config['licenses_group_url']='file:///usr/lib/ckan/default/src/ckanext-odsh/licenses.json'
try:
yield
finally:
config.clear()
config.update(_original_config)
...@@ -18,7 +18,6 @@ class TestUpload(helpers.FunctionalTestBase): ...@@ -18,7 +18,6 @@ class TestUpload(helpers.FunctionalTestBase):
def teardown(self): def teardown(self):
model.repo.rebuild_db() model.repo.rebuild_db()
@odsh_test() @odsh_test()
def test_upload_empty_form_fails(self): def test_upload_empty_form_fails(self):
# arrange # arrange
...@@ -75,17 +74,53 @@ class TestUpload(helpers.FunctionalTestBase): ...@@ -75,17 +74,53 @@ class TestUpload(helpers.FunctionalTestBase):
response5.mustcontain('odsh_temporal_start_not_date_error_label') response5.mustcontain('odsh_temporal_start_not_date_error_label')
response6.mustcontain('odsh_temporal_start_not_date_error_label') response6.mustcontain('odsh_temporal_start_not_date_error_label')
@odsh_test()
def test_upload_license_without_attribution_text(self):
# arrange
form = self._get_package_new_form()
form['title'] = 'Titel'
form['name'] = 'sdlkjsadlfkjas'
form['notes'] = 'notes'
form['owner_org'] = self.org['id']
form[self._get_field_name(
'spatial_uri')] = 'http://dcat-ap.de/def/politicalGeocoding/districtKey/01001'
form[self._get_field_name('issued')] = '2019-01-29'
form[self._get_field_name('temporal_start')] = '2019-01-29'
form[self._get_field_name('temporal_end')] = '2019-02-02'
form['license_id'] = 'http://dcat-ap.de/def/licenses/dl-zero-de/2.0'
# act
print(form)
form[self._get_field_key('licenseAttributionByText')
] = 'licenseAttributionByText'
form[self._get_field_name('licenseAttributionByText')] = None
response = self._submit_and_follow_form(form)
# assert
response.mustcontain('odsh_spatial_uri_unknown_error_label')
def _get_field_name(self, field): def _get_field_name(self, field):
checksum = odsh_create_checksum(field) checksum = odsh_create_checksum(field)
return 'extras__' + str(checksum) + '__value' return 'extras__' + str(checksum) + '__value'
def _get_field_key(self, field):
checksum = odsh_create_checksum(field)
return 'extras__' + str(checksum) + '__key'
def _submit_form(self, form): def _submit_form(self, form):
return webtest_submit(form, 'save', status=200, extra_environ=self.env) return webtest_submit(form, 'save', status=200, extra_environ=self.env)
def _submit_and_follow_form(self, form):
app = self._get_test_app()
# return webtest_submit(form, 'save', status=200, extra_environ=self.env)
return helpers.submit_and_follow(app, form, self.env, 'save')
def _get_package_new_form(self): def _get_package_new_form(self):
app = self._get_test_app() app = self._get_test_app()
user = factories.User() user = factories.User()
self.org = factories.Organization(
name="my-org",
users=[{'name': user['id'], 'capacity': 'admin'}]
)
self.env = {'REMOTE_USER': user['name'].encode('ascii')} self.env = {'REMOTE_USER': user['name'].encode('ascii')}
response = app.get( response = app.get(
url=url_for(controller='package', action='new'), url=url_for(controller='package', action='new'),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment