diff --git a/ckanext/odsh/helpers.py b/ckanext/odsh/helpers.py index de23276c836b57d4ff6495149a66216d60ad00a2..5f16c4a0be0bfa22ba8fc4fa5f2880c5719a5af4 100644 --- a/ckanext/odsh/helpers.py +++ b/ckanext/odsh/helpers.py @@ -11,6 +11,7 @@ from dateutil import parser from ckan.common import config import urllib import hashlib +import re get_action = logic.get_action log = logging.getLogger(__name__) @@ -104,25 +105,35 @@ def odsh_get_spatial_text(pkg_dict): return spatial return None +def extend_search_convert_local_to_utc_timestamp(str_timestamp): + if not str_timestamp: + return None + + if not re.match(r'\d\d\d\d-\d\d-\d\d', str_timestamp): + raise 'wrong format' + + dt = parser.parse(str_timestamp, dayfirst=False).isoformat() + + return dt+"Z" def odsh_render_datetime(datetime_, fromIso=True): date_format='{0.day:02d}.{0.month:02d}.{0.year:4d}' if not datetime_: return '' + if not re.match(r'\d\d\d\d-\d\d-\d\d', datetime_): + return '' try: if fromIso: DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S' else: DATETIME_FORMAT = '%Y-%m-%d' - dt = datetime.datetime.strptime( - datetime_, DATETIME_FORMAT) - return dt.strftime('%d.%m.%Y') - # dt = parser.parse(datetime_, dayfirst=False) - # return date_format.format(dt) + + dt = parser.parse(datetime_, dayfirst=False) + return date_format.format(dt) + except: return '' - def odsh_upload_known_formats(): value = config.get('ckanext.odsh.upload_formats', []) value = toolkit.aslist(value) diff --git a/ckanext/odsh/plugin.py b/ckanext/odsh/plugin.py index b2ade0381a5e9c6de61e6261727de560d888c0c7..ca6dc270b4adb035a1bcd6556ff087575d890175 100644 --- a/ckanext/odsh/plugin.py +++ b/ckanext/odsh/plugin.py @@ -144,13 +144,14 @@ def odsh_validate_extra_date(key, field, data, errors, context): if not ('id',) in data or data[('id',)][:7] != 'StaNord': raise toolkit.Invalid(field+':odsh_'+field+'_error_label') else: - try: - dt=parse(value, dayfirst=True) - _set_value(data, field, dt.isoformat()) - # datetime.datetime.strptime( - # value.split('T')[0], '%Y-%m-%d').isoformat() - except ValueError: - raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label') + if re.match(r'\d\d\d\d-\d\d-\d\d', value): + try: + dt=parse(value, dayfirst=True) + _set_value(data, field, dt.isoformat()) + return + except ValueError: + pass + raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label') def odsh_validate_extra_date_factory(field): @@ -363,20 +364,6 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm 'odsh_validate_temporal_end': odsh_validate_extra_date_factory('temporal_end'), 'odsh_tag_name_validator': odsh_tag_name_validator} - def extend_search_convert_local_to_utc_timestamp(self, str_timestamp): - DATETIME_FORMAT = '%Y-%m-%d' - if not str_timestamp: - return '' - - # Todo: do we need timezone conversions? - - local_datetime = datetime.datetime.strptime( - str_timestamp, DATETIME_FORMAT) - # tz_code = config.get('ckan.timezone', 'Australia/Melbourne') - # local = timezone(tz_code) - # utc_datetime = _make_aware(local_datetime, local) - # local_datetime = utc_datetime.astimezone(pytz.utc) - return local_datetime.strftime(DATETIME_FORMAT)+"T00:00:00Z" # Add the custom parameters to Solr's facet queries def before_search(self, search_params): @@ -389,11 +376,11 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm start_date=None end_date=None try: - start_date = self.extend_search_convert_local_to_utc_timestamp( + start_date = odsh_helpers.extend_search_convert_local_to_utc_timestamp( extras.get('ext_startdate')) - end_date = self.extend_search_convert_local_to_utc_timestamp( + end_date = odsh_helpers.extend_search_convert_local_to_utc_timestamp( extras.get('ext_enddate')) - except ValueError: + except: return search_params empty_range = start_date and end_date and start_date > end_date @@ -431,6 +418,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm fq = '{fq} ({start_query} OR {end_query} {enclosing_query})'.format( fq=fq, start_query=start_query, end_query=end_query, enclosing_query=enclosing_query) + print(fq) + # return modified facet queries search_params['fq'] = fq diff --git a/ckanext/odsh/tests/test_search.py b/ckanext/odsh/tests/test_search.py index 14d61acd23156320211c86bdcf7ca32852716a44..c25fd99585584e30d8d33ee220a3371f29a0e71e 100644 --- a/ckanext/odsh/tests/test_search.py +++ b/ckanext/odsh/tests/test_search.py @@ -52,6 +52,20 @@ class TestSearch(helpers.FunctionalTestBase): # assert assert dataset['name'] in response + @odsh_test() + def test_query_with_very_old_dataset(self): + # arrange + dataseta = self._create_dataset('do_not_find_me', '2011-01-01', '2013-12-31') + datasetb = self._create_dataset('old_dataset', '1111-01-01', '1860-12-31') + + # act + response = self._perform_date_search('1110-12-30', '1960-02-01') + + # assert + assert 'wrong_start_date_for_search' not in response + self._assert_datasets_in_response([datasetb], response) + self._assert_datasets_not_in_response([dataseta], response) + @odsh_test() def test_query_with_end_before_start_finds_no_dataset(self): # arrange