fix date hanlding

ckanext/odsh/helpers.py

@@ -11,6 +11,7 @@ from dateutil import parser
 from ckan.common import config
 import urllib
 import hashlib
+import re
 
 get_action = logic.get_action
 log = logging.getLogger(__name__)
@@ -104,25 +105,35 @@ def odsh_get_spatial_text(pkg_dict):
             return spatial
     return None
 
+def extend_search_convert_local_to_utc_timestamp(str_timestamp):
+    if not str_timestamp:
+        return None 
+
+    if not re.match(r'\d\d\d\d-\d\d-\d\d', str_timestamp):
+        raise 'wrong format'
+    
+    dt = parser.parse(str_timestamp, dayfirst=False).isoformat()
+
+    return dt+"Z"
 
 def odsh_render_datetime(datetime_, fromIso=True):
     date_format='{0.day:02d}.{0.month:02d}.{0.year:4d}'
     if not datetime_:
         return ''
+    if not re.match(r'\d\d\d\d-\d\d-\d\d', datetime_):
+        return ''
     try:
         if fromIso:
             DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
         else:
             DATETIME_FORMAT = '%Y-%m-%d'
-        dt = datetime.datetime.strptime(
-            datetime_, DATETIME_FORMAT)
-        return dt.strftime('%d.%m.%Y')
-        # dt = parser.parse(datetime_, dayfirst=False)
-        # return date_format.format(dt)
+
+        dt = parser.parse(datetime_, dayfirst=False)
+        return date_format.format(dt)
+
     except:
         return ''
 
-
 def odsh_upload_known_formats():
     value = config.get('ckanext.odsh.upload_formats', [])
     value = toolkit.aslist(value)

ckanext/odsh/plugin.py

@@ -144,12 +144,13 @@ def odsh_validate_extra_date(key, field, data, errors, context):
         if not ('id',) in data or data[('id',)][:7] != 'StaNord':
             raise toolkit.Invalid(field+':odsh_'+field+'_error_label')
     else:
+        if re.match(r'\d\d\d\d-\d\d-\d\d', value):
             try:
                 dt=parse(value, dayfirst=True)
                 _set_value(data, field, dt.isoformat())
-            # datetime.datetime.strptime(
-            #    value.split('T')[0], '%Y-%m-%d').isoformat()
+                return
             except ValueError:
+                pass
         raise toolkit.Invalid(field+':odsh_'+field+'_not_date_error_label')
 
 
@@ -363,20 +364,6 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
                 'odsh_validate_temporal_end': odsh_validate_extra_date_factory('temporal_end'),
                 'odsh_tag_name_validator': odsh_tag_name_validator}
 
-    def extend_search_convert_local_to_utc_timestamp(self, str_timestamp):
-        DATETIME_FORMAT = '%Y-%m-%d'
-        if not str_timestamp:
-            return ''
-
-        # Todo: do we need timezone conversions?
-
-        local_datetime = datetime.datetime.strptime(
-            str_timestamp, DATETIME_FORMAT)
-        # tz_code = config.get('ckan.timezone', 'Australia/Melbourne')
-        # local = timezone(tz_code)
-        # utc_datetime = _make_aware(local_datetime, local)
-        # local_datetime = utc_datetime.astimezone(pytz.utc)
-        return local_datetime.strftime(DATETIME_FORMAT)+"T00:00:00Z"
 
     # Add the custom parameters to Solr's facet queries
     def before_search(self, search_params):
@@ -389,11 +376,11 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         start_date=None
         end_date=None
         try:
-            start_date = self.extend_search_convert_local_to_utc_timestamp(
+            start_date = odsh_helpers.extend_search_convert_local_to_utc_timestamp(
                 extras.get('ext_startdate'))
-            end_date = self.extend_search_convert_local_to_utc_timestamp(
+            end_date = odsh_helpers.extend_search_convert_local_to_utc_timestamp(
                 extras.get('ext_enddate'))
-        except ValueError:
+        except:
             return search_params
 
         empty_range = start_date and end_date and start_date > end_date
@@ -431,6 +418,8 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         fq = '{fq} ({start_query} OR {end_query} {enclosing_query})'.format(
             fq=fq, start_query=start_query, end_query=end_query, enclosing_query=enclosing_query)
 
+        print(fq)
+
         # return modified facet queries
         search_params['fq'] = fq
 

ckanext/odsh/tests/test_search.py

@@ -52,6 +52,20 @@ class TestSearch(helpers.FunctionalTestBase):
         # assert
         assert dataset['name'] in response
 
+    @odsh_test()
+    def test_query_with_very_old_dataset(self):
+        # arrange
+        dataseta = self._create_dataset('do_not_find_me', '2011-01-01', '2013-12-31')
+        datasetb = self._create_dataset('old_dataset', '1111-01-01', '1860-12-31')
+
+        # act
+        response = self._perform_date_search('1110-12-30', '1960-02-01')
+
+        # assert
+        assert 'wrong_start_date_for_search' not in response
+        self._assert_datasets_in_response([datasetb], response)
+        self._assert_datasets_not_in_response([dataseta], response)
+
     @odsh_test()
     def test_query_with_end_before_start_finds_no_dataset(self):
         # arrange