From 822e344fc7ad8f6eb2d459cc63cff19b646f4916 Mon Sep 17 00:00:00 2001
From: Ngô Ngọc Đức Huy <huyngo@disroot.org>
Date: Mon, 13 May 2024 10:26:41 +0700
Subject: Add robots.txt

---
 config.yaml       |  1 +
 static/robots.txt | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 static/robots.txt

diff --git a/config.yaml b/config.yaml
index 5021db3..a828b19 100644
--- a/config.yaml
+++ b/config.yaml
@@ -2,6 +2,7 @@ baseURL: https://xrvs.net/
 copyright: CC-BY-SA 4.0
 defaultContentLanguage: en
 defaultContentLanguageInSubdir: true
+enableRobotsTXT: true
 languageCode: en-us
 languages:
   de:
diff --git a/static/robots.txt b/static/robots.txt
new file mode 100644
index 0000000..da6e670
--- /dev/null
+++ b/static/robots.txt
@@ -0,0 +1,67 @@
+# Dark Visitors Robots.txt
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
+User-agent: Bytespider
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/claudebot
+
+User-agent: ClaudeBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/diffbot
+
+User-agent: Diffbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
+User-agent: GPTBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/claude-web
+
+User-agent: Claude-Web
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/cohere-ai
+
+User-agent: cohere-ai
+Disallow: /
-- 
cgit 1.4.1