Added custom robot.txt file

CHANGELOG.md

+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- Custom robots.txt file.
+
+### Changed
+
+- Migrated controllers to blueprints (Pylons to Flask).
+- UI and webasset upgrades.
+- Nosetests changed to pytests.
+
+### Removed
+
+- Support for Python2 removed.
+
+## [1.4.3] - 2022-11-03
+
+### Added
+
+- Added "Musterdatenkatalog" references to RDF
+
+## [1.4.2] - 2022-10-24
+
+## [1.4.1] - 2022-08-17
+
+## [1.4.0] - 2022-08-11

ckanext/odsh/plugin.py

@@ -53,6 +53,7 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
         rules = [
             ('/info_page', 'info_page', default.info_page),
             ('/home', 'start', default.start),
+            ('/robots.txt', 'robots_txt', default.robots_txt),
             ('/not_found', 'not_found', default.not_found), ]
         for rule in rules:
             bp_default.add_url_rule(*rule)

ckanext/odsh/templates/robots.txt

+User-agent: SemrushBot
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: SEOkicks
+Disallow: /
+
+User-agent: *
+Disallow: /dataset/rate/
+Disallow: /revision/
+Disallow: /dataset/*/history
+Disallow: /api/
+Crawl-Delay: 10
\ No newline at end of file

ckanext/odsh/views/default.py

@@ -17,3 +17,7 @@ def start():
 def not_found():
     log.info("views.default::not_found")
     toolkit.abort(404)
+
+def robots_txt():
+    log.info("views.default::robots_txt")
+    return toolkit.render(u'robots.txt')