diff options
| author | Ngô Ngọc Đức Huy <huyngo@disroot.org> | 2024-05-13 10:26:41 +0700 |
|---|---|---|
| committer | Ngô Ngọc Đức Huy <huyngo@disroot.org> | 2024-05-13 10:26:41 +0700 |
| commit | 822e344fc7ad8f6eb2d459cc63cff19b646f4916 (patch) | |
| tree | ef6d46082256cf15c1a447468466a000bb22a2e9 | |
| parent | 1c5529344cc3a36d3afef584d3d58e9d3e5552f4 (diff) | |
| download | blog-822e344fc7ad8f6eb2d459cc63cff19b646f4916.tar.gz | |
Add robots.txt
| -rw-r--r-- | config.yaml | 1 | ||||
| -rw-r--r-- | static/robots.txt | 67 |
2 files changed, 68 insertions, 0 deletions
diff --git a/config.yaml b/config.yaml index 5021db3..a828b19 100644 --- a/config.yaml +++ b/config.yaml @@ -2,6 +2,7 @@ baseURL: https://xrvs.net/ copyright: CC-BY-SA 4.0 defaultContentLanguage: en defaultContentLanguageInSubdir: true +enableRobotsTXT: true languageCode: en-us languages: de: diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..da6e670 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,67 @@ +# Dark Visitors Robots.txt + +# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/claudebot + +User-agent: ClaudeBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/diffbot + +User-agent: Diffbot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + +User-agent: GPTBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/omgili + +User-agent: omgili +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/claude-web + +User-agent: Claude-Web +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai +Disallow: / |