From 3a005f23696ad17d02582ec0e2d972620b999ab1 Mon Sep 17 00:00:00 2001 From: yyamashita Date: Thu, 14 May 2026 23:29:56 +0900 Subject: Add crawler bypass header to skip X-Robots-Tag Requests with X-Crawler-Access header and the correct token will have the X-Robots-Tag response header removed, allowing trusted crawlers to index the site. Co-Authored-By: Claude Sonnet 4.6 --- Caddyfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Caddyfile b/Caddyfile index fa8e706..7e05478 100644 --- a/Caddyfile +++ b/Caddyfile @@ -1,12 +1,20 @@ golive.yyamashita.com { header Referrer-Policy "no-referrer" header X-Robots-Tag "noindex, nofollow" + + @crawler_bypass header X-Crawler-Access "f9a3c1e847b2d056" + header @crawler_bypass -X-Robots-Tag + reverse_proxy tokyo-app:3000 } whoisband.yyamashita.com { header Referrer-Policy "no-referrer" header X-Robots-Tag "noindex, nofollow" + + @crawler_bypass header X-Crawler-Access "f9a3c1e847b2d056" + header @crawler_bypass -X-Robots-Tag + reverse_proxy whois-app:3000 } code.yyamashita.com { -- cgit v1.2.3