1 files changed, 8 insertions(+), 0 deletions(-)

M roles/srht/files/srht_robots.txt

@@ 4,6 4,8 @@ Disallow: /*.tar.gz$
 Disallow: /metrics
 Disallow: /*/*/blame/*
 Disallow: /*/*/log/*
+Disallow: /*/*/tree/*
+Disallow: /*/*/item/*
 
 User-agent: SemrushBot
 Disallow: /


@@ 43,3 45,9 @@ Disallow: /
 
 User-agent: turnitinbot
 Disallow: /
+
+User-agent: Turnitin
+Disallow: /
+
+User-agent: Seekport Crawler
+Disallow: /