From c0c824a01e13ca4a14f3626178d842edcba47361 Mon Sep 17 00:00:00 2001
From: John Bowdre <john@bowdre.net>
Date: Fri, 12 Apr 2024 15:29:07 -0500
Subject: [PATCH] let ai bots know they aren't welcome

---
 config/_default/hugo.toml   |  1 +
 config/_default/params.toml | 29 +++++++++++++++++++++++++++++
 layouts/robots.txt          |  8 ++++++++
 3 files changed, 38 insertions(+)
 create mode 100644 layouts/robots.txt

diff --git a/config/_default/hugo.toml b/config/_default/hugo.toml
index 95426b9..a8c8ff4 100644
--- a/config/_default/hugo.toml
+++ b/config/_default/hugo.toml
@@ -6,6 +6,7 @@ paginate = 10
 languageCode = "en"
 DefaultContentLanguage = "en"
 enableInlineShortcodes = true
+enableRobotsTXT = true
 
 # define gemini media type
 [mediaTypes]
diff --git a/config/_default/params.toml b/config/_default/params.toml
index 0f902ff..e935b8c 100644
--- a/config/_default/params.toml
+++ b/config/_default/params.toml
@@ -8,6 +8,35 @@ numberOfRelatedPosts = 5
 
 indexTitle = ".-. ..- -. - .. -- . - . .-. .-. --- .-."
 
+robots = [
+  "AdsBot-Google",
+  "Amazonbot",
+  "anthropic-ai",
+  "Applebot",
+  "AwarioRssBot",
+  "AwarioSmartBot",
+  "Bytespider",
+  "CCBot",
+  "ChatGPT",
+  "ChatGPT-User",
+  "Claude-Web",
+  "ClaudeBot",
+  "cohere-ai",
+  "DataForSeoBot",
+  "Diffbot",
+  "FacebookBot",
+  "FacebookBot",
+  "Google-Extended",
+  "GPTBot",
+  "ImagesiftBot",
+  "magpie-crawler",
+  "omgili",
+  "Omgilibot",
+  "peer39_crawler",
+  "PerplexityBot",
+  "YouBot"
+]
+
 # Comments
 comments = true
 giscusCategory = "Announcements"
diff --git a/layouts/robots.txt b/layouts/robots.txt
new file mode 100644
index 0000000..693c133
--- /dev/null
+++ b/layouts/robots.txt
@@ -0,0 +1,8 @@
+Sitemap: {{ .Site.BaseURL }}/sitemap.xml
+
+User-agent: *
+Disallow:
+{{ range .Site.Params.robots }}
+User-agent: {{ . }}
+{{- end }}
+Disallow: /