From 82c85a7457ba20d8bda3941456ba413ae6dce7ee Mon Sep 17 00:00:00 2001
From: anonymous <anonymous>
Date: Tue, 18 Jun 2019 16:29:05 +0200
Subject: [PATCH] adds a label with text NEW to new datasets
---
ckanext/odsh/helpers.py | 27 ++
.../odsh/i18n/de/LC_MESSAGES/ckanext-odsh.mo | Bin 8305 -> 8329 bytes
.../odsh/i18n/de/LC_MESSAGES/ckanext-odsh.po | 5 +-
ckanext/odsh/plugin.py | 29 +++
ckanext/odsh/public/odsh.css | 8 +
.../odsh/templates/snippets/package_item.html | 3 +
validation.py | 241 ++++++++++++++++++
7 files changed, 312 insertions(+), 1 deletion(-)
create mode 100644 validation.py
diff --git a/ckanext/odsh/helpers.py b/ckanext/odsh/helpers.py
index 1fe0abf4..4acbb190 100644
--- a/ckanext/odsh/helpers.py
+++ b/ckanext/odsh/helpers.py
@@ -266,3 +266,30 @@ def odsh_is_slave():
if c is None or (c != 'True' and c != 'False'):
return -1
return 1 if c == 'True' else 0
+
+
+def is_within_last_month(date, date_ref=None):
+ '''
+ date is a datetime.date object containing the date to be checked
+ date_ref is a datetime.date object containing the reference date
+ if date_ref is not specified, the date of today is used
+ this method is needed by the method OdshPlugin.before_view in plugin.py
+ '''
+
+ if not date_ref:
+ date_ref = datetime.date.today()
+
+ [year_ref, month_ref, day_ref] = [date_ref.year, date_ref.month, date_ref.day]
+
+ try:
+ if month_ref > 1:
+ one_month_ago = datetime.date(year_ref, month_ref-1, day_ref)
+ else:
+ one_month_ago = datetime.date(year_ref-1, 12, day_ref)
+ except ValueError:
+ # this happens if month before month_ref has less days than month_ref
+ one_month_ago = datetime.date(year_ref, month_ref, 1) - datetime.timedelta(days=1)
+
+ if date > one_month_ago:
+ return True
+ return False
diff --git a/ckanext/odsh/i18n/de/LC_MESSAGES/ckanext-odsh.mo b/ckanext/odsh/i18n/de/LC_MESSAGES/ckanext-odsh.mo
index ab0a6177c3cc3782e3ad5f9f36bd1a9e3e1f5af1..293456c5b0934cc0dd8b703e467bce532a80f134 100644
GIT binary patch
delta 2237
zcmez9(CJuzPl#nI0|Ns?I|BoQ3<Cp02n&dZz&#uc49pA+482f#B9xxW!N34gIupvD
z$HBlL#K6F?h=YNFpMim4D+dDuHv<F1Q7C<p17iMN4h9BR1_p*-91INX3=H)Q44e!M
z;tUK79Gnmh3Q$^y6Jn4BCj$c)0|SExCj)~Z0|P@aCj)~50|P@UCj)~d0|P@FRD1zc
z-6p91y_^t>PI5vVcAk@gfsKKI;Snd;N`{x53=9$=`#(b^c(@oC1Q-|?gt;IVsc<nc
zh%qoQm~nw!%HYcdaY!f^#NY%jh=FNb3=Dh>3=Fwk3=E<S3=B0~3=FCa3=EUFAR)Jd
zi-Cccfq`KU7XyO|0|Ub;E(Qh>1_p-rTnr4d^$ZLQEZh(UO56}18E``k^x}qC6ba?W
zb3-)daYKC245fRyAt5)N8xmrRxj{Z-VA#yfz+lb5z;K3}fk6rs!aR^DQ|4h{;9+23
z(BNTUs0X>+l!t*qmw|!7g$Lq;A|8mv9Xt#S91IK$eLN7KPUnF*Y#vm6Ee|B>wm}U(
z#{+S|WhnnKRQ(I6_;((N!`XQu<_hvM)Poa|6feYO@=#iZ7ZTK(ybudbpyJL@+5<}a
z@-i?eGB7X%LHQ-Tkf^GJTF?lkyP)Eeq3UMyLOis9w;mGo%XuL|wiT-294{nDFY_`m
zXfrS{+~<V^B{v@=PTlz+X~dfk;*(fDhzHX67#M^Z7#OnoAR!3~UQptm$_GibYxy8)
z;~XCYgC+w5!xxYOP-2nehbVC2XJC*6rEVy{g`a^zmVtp`DL*7_9Oh?WkY`|ExC`a~
z<%d`-E&xg8G6E33z5v7=69I_Ff}!#`Q2BZRh(qgJpbBRQKwP|C0OF9d0+6_RC&0kK
z#K6Gt1FG@20K_4nyeQ4Uz#t{az@P`pC4vkLULc1HGB8*$FfeQoWMG)U$iVPY5E8;w
z!jQDlAq<Jq84$XjVYx6QsJ9726dn?Wq|OV%5C`2AhWOyKFeK=BL?G&nL?G%ML>L%A
z4)KNZGesa4l!-v}w~9cr@oXr6lL!NY8Uq8v2@z08)-y1?hZ@K)3eg}d3NgS~6q1;n
zL?PvZmnbCBM2kWaXObu+$aA6m22lnEWd;U@ZczpXB?bnD^`ej{xCGVr5la6Mh2*aP
zAU-Jnvx-6THIEp?CyHW_L}x1oNo<*75C`RoK@6-BgA}E+#Xvq_VAvrB3Bem;khJnd
z43apR#39*JL>%HV9Vp*P9O9uUaR!EZP|1`n4vFhZaY)=YLp975XJ80nU|`rR4oR&X
z640QRfFvFV35dZV5)2GR3=9le5)cQ?m4NtguLQ*0!xE5?I}7Dsk$@!9d!X_c6c>*r
z7#M6B7#O%DAtB)*2?_dGNr+EUB_S>@mxLJD0i`ENLVP$!65{YJP<`hlAr8F;)&E2i
z;(%|GkP!VV35g;`sd@$m9R>yl6)8xZ21-HlbBq+kqEslI52Y)mAU<xEf~4k|Qjn5!
zr4$2$IRgX3btwi00|o{Lerbq<9i$;q7AOtLB?Z!ukeM$H@z~OOX-Fd4E)8+vDX7Ma
z(hLk5pxR0rVga`d!~#nhh=GMN5CbR3Kyt-?8A#kem4QSBvn(W)o5@1-$I3#|RJSYx
zgCPS0!**GSN9te7LW2CiEX2hkau9=5<ro;W7#J9w<se0Do*cxX6XYO?Z?PQ2-~)0H
zhrN@77_24_DJdi5At5<M9unkp<snhJP##hlt&|6wBgJq;o`J!Vfq~(nJOhI>0|SG!
n0wl4;Z?5DpU}AC&-@Ke_7YDPsp26l6;ZICVuA!Sl#ALYuYk|gz
delta 2215
zcmeBl{OC}BPl#nI0|Ns?8v_G_3<Cp$4-1Hgz%?8U49pA+47E_Y5lXjmFff3Wc0&1m
z91ILX3=9mDI2aiC85kIraxgG(GcYi0h0+H(Am*RtU|?WnU|@K~!N9=Iz);WdfrEiT
zoPmMi2UMW|Cq$zRC&VBPP6h@p1_lNTP6h@+1_lOaP6h@A1_p*uP6h@^1_p*QsQ3h^
zx<yd^YdIkn?c{_wY(FOh0~-Sa!zE4z22KVBhMSxW3=$0W3=EH<5`Q=u7z7v?7?`;r
z7Kv~%Fo-cQFsN~XUCLm~1#yTg7sTKIE{K6)Tnr3+3=9mhTnr4N3=9l8Tnr4V3=9lS
zTnr4n3=9k_xEL5z7#J9~aWOE6FfcIO=3-!wWnf_V0Ftj~U|`_mhWJE=8)A?dH^c&0
zDBqhKqA`ja;_`ebUC9jzsb+3SNcD4re8j*opPPZfnt_2~2R8$Q6axdp7j8&Y@$)b+
z@Gvkii10w{QRHD@sMlp+U@+i;xIB&rVsQx%0|N&G149)L#D~pL4P8+2nLLmvTf_q~
zcoz@E0sEo+^HB9ypyH2tAP)Zw)&Cc&j*XY09-N4{c_9V~@Ir!Elo#Sc1*o__ls19V
zmb?rMiVO@4c2Ir-FC>amc_9wZh0<kE@p`Dbc3y~wdUzpGGnuy@5@ZXZ3U={Af^<JG
z1A{gL1H)-vNKk&~h2#QbK1kXy=Yv?}$p>+G5FZ1BFarZaI3FY=v-uz))W`=(tTXu_
zX<`>21A`_51H%I-pR1l9qQHQkfkBRefgupeFW`qHl8O9~w6UI_fkB>uf#D>S|B@eK
zF|z<9b#n+n_|gIpa})$19<ztaM?mGX1t1PBfU2u+5rDXOu>iy&I|U$dbxVMOfr){E
z;R#gZa{-7$-Uu)-NHZ`nun96S=rJ%bC<!t!c!3-)$iQI1z`!s^kbz+WBLl-#K}Zm%
z2}9CGi7+HeTcGr0VQ@&-Gb|E@C|oBDNu7IyAr3kz4DrEzVMx&ZfU1)dfvD3FfjGnx
z$`2KRSdb(F(O)P6$-eDS{yY%|1~mo-hAkqHki0DdvbUar;WtzPrzpe#c~MAW(i4T0
z2WFy>MB^?BNt}M7kRXqQ@^eHP7?c?p7|KN%7?c<o7-ow?qF^6X-(4vEL==*{UPJjG
zMHv`GK>7cND8whcVvt0qDF$&!s2Iec6fsEA*(?Te`2sO;&@mhkgQSr&Vvt1nN(_?d
z{)j<BKtvqE*Aj<##7P|FV+Mv`aYz&=iZd|OgNo2>afpUiaR!DEkPpNmsq_O>gOmg$
z(P&6O)Y(WdFc>i~Fa$|J9MmcS@!3)dh{3BRAR)FD%HJaaNu0+dAW?8yf`P%7fq~%@
zsQd*Li3XC8Aa|97_{3Kd;^KHoh=B!Ax=Iq_qZUa>Vw?ljw@nh_vwcwgXCxsGxF-q8
zwofG?QS(xgfkB6Xfq_Q~5|x%x^^kn*A_cL?7fOdi=|m}rkF%v9skl*!fdN!vPLyI`
zFlS(3*e}JvV8Fn@@LdYxU=3+Vlvzqcaz}(TBxKs9As*|MhNPYO(hvu3s+Wdn+$qh#
zpuxbva7r3t!DnfR1&T5d10!W123E*Ga>H^NNZg;5fkee?8A#B|$wKtI%0kjou`C0F
zAp--$d|8M`F3LjclIOAzd+YzmLJZ=SV_?u?U|`UegA}b{auAnR$Uzccw;aUa6><=V
zU6X?t%qI^iAsyr)Az3323G!BXNR)QULn@((@(^>@$TKilf(kNu1_ozPEHEoT5}W(x
cP7VX6&3n0aa4;L|nQX2Q{=~F7PfVH%0D6wZVE_OC
diff --git a/ckanext/odsh/i18n/de/LC_MESSAGES/ckanext-odsh.po b/ckanext/odsh/i18n/de/LC_MESSAGES/ckanext-odsh.po
index b09176f7..7357bf88 100644
--- a/ckanext/odsh/i18n/de/LC_MESSAGES/ckanext-odsh.po
+++ b/ckanext/odsh/i18n/de/LC_MESSAGES/ckanext-odsh.po
@@ -419,4 +419,7 @@ msgid "Create New Data Request"
msgstr "Neuen Datensatz vorschlagen"
msgid "Suggest New Data Request"
-msgstr "Datensatz vorschlagen"
\ No newline at end of file
+msgstr "Datensatz vorschlagen"
+
+msgid "NEW"
+msgstr "NEU"
\ No newline at end of file
diff --git a/ckanext/odsh/plugin.py b/ckanext/odsh/plugin.py
index b8f211af..c27478a9 100644
--- a/ckanext/odsh/plugin.py
+++ b/ckanext/odsh/plugin.py
@@ -525,4 +525,33 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
self.map_qa_score(dict_pkg)
return dict_pkg
+
+
+ # IPackageController
+
+ def before_view(self, pkg_dict):
+ '''
+ add a key 'is_new' to pkg_dict
+ the value for this key is True if the dataset has been modified within the last month
+ the value is used in the snippet package_item.html
+ '''
+ is_new = self._is_package_new(pkg_dict)
+ pkg_dict.update({'is_new':is_new})
+ return pkg_dict
+
+ def _is_package_new(self, pkg_dict):
+ date_last_modified = self._get_date_from_string(pkg_dict['metadata_modified'])
+ is_new = odsh_helpers.date_checker.is_within_last_month(date_last_modified)
+ return is_new
+
+ def _get_date_from_string(self, date_time_str):
+ # todo: update this function if used in different context
+ date_time_format = '%Y-%m-%dT%H:%M:%S.%f' #e.g. u'2019-06-12T11:56:25.059563'
+ try:
+ date_time = datetime.datetime.strptime(date_time_str, date_time_format)
+ except ValueError:
+ # if date cannot be converted from string fall back to 1.1.2000
+ date = datetime.date(2000, 1, 1)
+ date = date_time.date()
+ return date
diff --git a/ckanext/odsh/public/odsh.css b/ckanext/odsh/public/odsh.css
index 893d2a64..e5b8e4ee 100644
--- a/ckanext/odsh/public/odsh.css
+++ b/ckanext/odsh/public/odsh.css
@@ -1979,4 +1979,12 @@ p.package-info-categorie
.datarequest-item-autor-name
{
padding-left: 4px;
+}
+
+.new-dataset-label
+{
+ background-color: #d4004b!important;;
+ padding: 3px 3px 1px 3px;
+ font-size: 14px;
+ margin-right: 4px;
}
\ No newline at end of file
diff --git a/ckanext/odsh/templates/snippets/package_item.html b/ckanext/odsh/templates/snippets/package_item.html
index ff1ef8d4..d1a84c3e 100644
--- a/ckanext/odsh/templates/snippets/package_item.html
+++ b/ckanext/odsh/templates/snippets/package_item.html
@@ -43,6 +43,9 @@ Example:
<p>{{org}}</p>
<h3 class="dataset-heading">
{% block heading_private %}
+ {% if package.is_new %}
+ <span class='label new-dataset-label'>{{ _('NEW') }}</span>
+ {% endif %}
{% if package.private %}
<span class="dataset-private label label-inverse">
<i class="fa fa-lock"></i>
diff --git a/validation.py b/validation.py
new file mode 100644
index 00000000..e50cbb43
--- /dev/null
+++ b/validation.py
@@ -0,0 +1,241 @@
+# This Python file uses the following encoding: utf-8
+import logging
+import csv
+import re
+import urllib2
+import json
+from itertools import count
+from dateutil.parser import parse
+
+import ckan.plugins.toolkit as toolkit
+import ckan.model as model
+from ckan.lib.navl.dictization_functions import Missing
+
+from pylons import config
+
+import pdb
+
+_ = toolkit._
+
+log = logging.getLogger(__name__)
+
+
+def _extract_value(data, field):
+ key = None
+ for k in data.keys():
+ if data[k] == field:
+ key = k
+ break
+ if key is None:
+ return None
+ return data[(key[0], key[1], 'value')]
+
+
+def validate_extra_groups(data, requireAtLeastOne, errors):
+ value = _extract_value(data, 'groups')
+ if value != None:
+ # 'value != None' means the extra key 'groups' was found,
+ # so the dataset came from manual editing via the web-frontend.
+ if not value:
+ if requireAtLeastOne:
+ errors['groups'] = 'at least one group needed'
+ data[('groups', 0, 'id')] = ''
+ return
+
+ groups = [g.strip() for g in value.split(',') if value.strip()]
+ for k in data.keys():
+ if len(k) == 3 and k[0] == 'groups':
+ data[k] = ''
+ # del data[k]
+ if len(groups) == 0:
+ if requireAtLeastOne:
+ errors['groups'] = 'at least one group needed'
+ return
+
+ for num, group in zip(range(len(groups)), groups):
+ data[('groups', num, 'id')] = group
+ else: # no extra-field 'groups'
+ # dataset might come from a harvest process
+ if not data.get(('groups', 0, 'id'), False) and \
+ not data.get(('groups', 0, 'name'), False):
+ errors['groups'] = 'at least one group needed'
+
+
+def validate_extras(key, data, errors, context):
+ extra_errors = {}
+ isStaNord = ('id',) in data and data[('id',)][:7] == 'StaNord'
+
+ validate_extra_groups(data, True, extra_errors)
+ validate_extra_date_new(key, 'issued', data, isStaNord, extra_errors)
+ validate_extra_date_new(key, 'temporal_start',
+ data, isStaNord, extra_errors)
+ validate_extra_date_new(key, 'temporal_end', data, True, extra_errors)
+
+ if len(extra_errors.values()):
+ raise toolkit.Invalid(extra_errors)
+
+
+def _set_value(data, field, value):
+ key = None
+ for k in data.keys():
+ if data[k] == field:
+ key = k
+ break
+ if key is None:
+ return None
+ data[(key[0], key[1], 'value')] = value
+
+
+def validate_extra_date_new(key, field, data, optional, errors):
+ value = _extract_value(data, field)
+
+ if not value:
+ if not optional:
+ errors[field] = 'empty'
+ return
+ else:
+ if re.match(r'\d\d\d\d-\d\d-\d\d', value):
+ try:
+ dt = parse(value)
+ _set_value(data, field, dt.isoformat())
+ return
+ except ValueError:
+ pass
+ errors[field] = 'not a valid date'
+
+
+def validate_licenseAttributionByText(key, data, errors, context):
+ register = model.Package.get_license_register()
+ isByLicense = False
+ for k in data:
+ if len(k) > 0 and k[0] == 'license_id' and data[k] and not isinstance(data[k], Missing) and \
+ 'Namensnennung' in register[data[k]].title:
+ isByLicense = True
+ break
+ hasAttribution = False
+ for k in data:
+ if data[k] == 'licenseAttributionByText':
+ if isinstance(data[(k[0], k[1], 'value')], Missing) or (k[0], k[1], 'value') not in data:
+ del data[(k[0], k[1], 'value')]
+ del data[(k[0], k[1], 'key')]
+ break
+ else:
+ value = data[(k[0], k[1], 'value')]
+ hasAttribution = value != ''
+ break
+ if not hasAttribution:
+ current_indexes = [k[1] for k in data.keys()
+ if len(k) > 1 and k[0] == 'extras']
+
+ new_index = max(current_indexes) + 1 if current_indexes else 0
+ data[('extras', new_index, 'key')] = 'licenseAttributionByText'
+ data[('extras', new_index, 'value')] = ''
+
+ if isByLicense and not hasAttribution:
+ raise toolkit.Invalid(
+ 'licenseAttributionByText: empty not allowed')
+
+ if not isByLicense and hasAttribution:
+ raise toolkit.Invalid(
+ 'licenseAttributionByText: text not allowed for this license')
+
+
+def known_spatial_uri(key, data, errors, context):
+ value = _extract_value(data, 'spatial_uri')
+
+ if not value:
+ poly = None
+
+ # some harvesters might import a polygon directly...
+ # pdb.set_trace()
+ poly = _extract_value(data, 'spatial')
+
+ has_old_uri = False
+ pkg = context.get('package', None)
+ if pkg:
+ old_uri = pkg.extras.get('spatial_uri', None)
+ has_old_uri = old_uri != None and len(old_uri) > 0
+ if not poly:
+ poly = pkg.extras.get('spatial', None)
+ if not poly or has_old_uri:
+ raise toolkit.Invalid('spatial_uri: empty not allowed')
+ else:
+ if poly:
+ new_index = next_extra_index(data)
+ data[('extras', new_index+1, 'key')] = 'spatial'
+ data[('extras', new_index+1, 'value')] = poly
+ return
+
+ mapping_file = config.get('ckanext.odsh.spatial.mapping')
+ try:
+ mapping_file = urllib2.urlopen(mapping_file)
+ except Exception:
+ raise Exception("Could not load spatial mapping file!")
+
+ not_found = True
+ spatial_text = str()
+ spatial = str()
+ cr = csv.reader(mapping_file, delimiter="\t")
+ for row in cr:
+ if row[0].encode('UTF-8') == value:
+ not_found = False
+ spatial_text = row[1]
+ loaded = json.loads(row[2])
+ spatial = json.dumps(loaded['geometry'])
+ break
+ if not_found:
+ raise toolkit.Invalid(
+ 'spatial_uri: uri unknown')
+
+ new_index = next_extra_index(data)
+
+ data[('extras', new_index, 'key')] = 'spatial_text'
+ data[('extras', new_index, 'value')] = spatial_text
+ data[('extras', new_index+1, 'key')] = 'spatial'
+ data[('extras', new_index+1, 'value')] = spatial
+
+
+def next_extra_index(data):
+ current_indexes = [k[1] for k in data.keys()
+ if len(k) > 1 and k[0] == 'extras']
+
+ return max(current_indexes) + 1 if current_indexes else 0
+
+
+def tag_name_validator(value, context):
+ tagname_match = re.compile('[\w \-.\:\(\)\ยด\`]*$', re.UNICODE)
+ if not tagname_match.match(value):
+ raise toolkit.Invalid(_('Tag "%s" must be alphanumeric '
+ 'characters or symbols: -_.:()') % (value))
+ return value
+
+
+def tag_string_convert(key, data, errors, context):
+ '''Takes a list of tags that is a comma-separated string (in data[key])
+ and parses tag names. These are added to the data dict, enumerated. They
+ are also validated.'''
+ if isinstance(data[key], basestring):
+ tags = [tag.strip()
+ for tag in data[key].split(',')
+ if tag.strip()]
+ else:
+ tags = data[key]
+
+ current_index = max([int(k[1]) for k in data.keys()
+ if len(k) == 3 and k[0] == 'tags'] + [-1])
+
+ for num, tag in zip(count(current_index+1), tags):
+ data[('tags', num, 'name')] = tag
+
+ for tag in tags:
+ toolkit.get_validator('tag_length_validator')(tag, context)
+ tag_name_validator(tag, context)
+
+
+def get_validators():
+ return {
+ 'known_spatial_uri': known_spatial_uri,
+ 'odsh_tag_name_validator': tag_name_validator,
+ 'odsh_validate_extras': validate_extras,
+ 'validate_licenseAttributionByText': validate_licenseAttributionByText
+ }
--
GitLab