From d69a32d79fb826cbc5100152f43b4e63ca357db5 Mon Sep 17 00:00:00 2001
From: Thorge Petersen <petersen@rz.uni-kiel.de>
Date: Fri, 2 Jun 2023 14:43:20 +0200
Subject: [PATCH] Added custom robot.txt file
---
CHANGELOG.md | 34 +++++++++++++++++++++++++++++++
ckanext/odsh/plugin.py | 1 +
ckanext/odsh/templates/robots.txt | 18 ++++++++++++++++
ckanext/odsh/views/default.py | 4 ++++
4 files changed, 57 insertions(+)
create mode 100644 CHANGELOG.md
create mode 100644 ckanext/odsh/templates/robots.txt
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000..68232174
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,34 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- Custom robots.txt file.
+
+### Changed
+
+- Migrated controllers to blueprints (Pylons to Flask).
+- UI and webasset upgrades.
+- Nosetests changed to pytests.
+
+### Removed
+
+- Support for Python2 removed.
+
+## [1.4.3] - 2022-11-03
+
+### Added
+
+- Added "Musterdatenkatalog" references to RDF
+
+## [1.4.2] - 2022-10-24
+
+## [1.4.1] - 2022-08-17
+
+## [1.4.0] - 2022-08-11
diff --git a/ckanext/odsh/plugin.py b/ckanext/odsh/plugin.py
index b270fc0b..0539798a 100644
--- a/ckanext/odsh/plugin.py
+++ b/ckanext/odsh/plugin.py
@@ -53,6 +53,7 @@ class OdshPlugin(plugins.SingletonPlugin, DefaultTranslation, DefaultDatasetForm
rules = [
('/info_page', 'info_page', default.info_page),
('/home', 'start', default.start),
+ ('/robots.txt', 'robots_txt', default.robots_txt),
('/not_found', 'not_found', default.not_found), ]
for rule in rules:
bp_default.add_url_rule(*rule)
diff --git a/ckanext/odsh/templates/robots.txt b/ckanext/odsh/templates/robots.txt
new file mode 100644
index 00000000..c3239621
--- /dev/null
+++ b/ckanext/odsh/templates/robots.txt
@@ -0,0 +1,18 @@
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: SEOkicks
+Disallow: /
+
+User-agent: *
+Disallow: /dataset/rate/
+Disallow: /revision/
+Disallow: /dataset/*/history
+Disallow: /api/
+Crawl-Delay: 10
\ No newline at end of file
diff --git a/ckanext/odsh/views/default.py b/ckanext/odsh/views/default.py
index 0058076e..aee7f4f8 100644
--- a/ckanext/odsh/views/default.py
+++ b/ckanext/odsh/views/default.py
@@ -17,3 +17,7 @@ def start():
def not_found():
log.info("views.default::not_found")
toolkit.abort(404)
+
+def robots_txt():
+ log.info("views.default::robots_txt")
+ return toolkit.render(u'robots.txt')
--
GitLab