From 640b5b66c5a0e47cd4f99ab3f267a18cc21f53c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorben=20G=C3=BCnther?= Date: Mon, 20 Feb 2023 12:39:20 +0100 Subject: [PATCH] srht: Update robots.txt --- roles/srht/files/srht_robots.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/srht/files/srht_robots.txt b/roles/srht/files/srht_robots.txt index da5b88f..c41bd9e 100644 --- a/roles/srht/files/srht_robots.txt +++ b/roles/srht/files/srht_robots.txt @@ -4,6 +4,8 @@ Disallow: /*.tar.gz$ Disallow: /metrics Disallow: /*/*/blame/* Disallow: /*/*/log/* +Disallow: /*/*/tree/* +Disallow: /*/*/item/* User-agent: SemrushBot Disallow: / @@ -43,3 +45,9 @@ Disallow: / User-agent: turnitinbot Disallow: / + +User-agent: Turnitin +Disallow: / + +User-agent: Seekport Crawler +Disallow: / -- 2.44.0