diff --git a/ckanext/odsh/templates/home/robots.txt b/ckanext/odsh/templates/home/robots.txt index 84768487c477570b6dc3baef028ab08a0fd3b24e..762f03a557f763b0ab3ad6af509cb49db8440fe3 100644 --- a/ckanext/odsh/templates/home/robots.txt +++ b/ckanext/odsh/templates/home/robots.txt @@ -1,12 +1,8 @@ -User-agent: SemrushBot -Disallow: / - -User-agent: PetalBot -Disallow: / - +User-agent: AwarioRssBot +User-agent: AwarioSmartBot User-agent: DataForSeoBot -Disallow: / - +User-agent: PetalBot +User-agent: SemrushBot User-agent: SEOkicks Disallow: /