let ai bots know they aren't welcome

config/_default/hugo.toml

@@ -6,6 +6,7 @@ paginate = 10
 languageCode = "en"
 DefaultContentLanguage = "en"
 enableInlineShortcodes = true
+enableRobotsTXT = true
 
 # define gemini media type
 [mediaTypes]

config/_default/params.toml

@@ -8,6 +8,35 @@ numberOfRelatedPosts = 5
 
 indexTitle = ".-. ..- -. - .. -- . - . .-. .-. --- .-."
 
+robots = [
+  "AdsBot-Google",
+  "Amazonbot",
+  "anthropic-ai",
+  "Applebot",
+  "AwarioRssBot",
+  "AwarioSmartBot",
+  "Bytespider",
+  "CCBot",
+  "ChatGPT",
+  "ChatGPT-User",
+  "Claude-Web",
+  "ClaudeBot",
+  "cohere-ai",
+  "DataForSeoBot",
+  "Diffbot",
+  "FacebookBot",
+  "FacebookBot",
+  "Google-Extended",
+  "GPTBot",
+  "ImagesiftBot",
+  "magpie-crawler",
+  "omgili",
+  "Omgilibot",
+  "peer39_crawler",
+  "PerplexityBot",
+  "YouBot"
+]
+
 # Comments
 comments = true
 giscusCategory = "Announcements"

layouts/robots.txt

@@ -0,0 +1,8 @@
+Sitemap: {{ .Site.BaseURL }}/sitemap.xml
+
+User-agent: *
+Disallow:
+{{ range .Site.Params.robots }}
+User-agent: {{ . }}
+{{- end }}
+Disallow: /