Add crawler bypass header to skip X-Robots-Tag

Requests with X-Crawler-Access header and the correct token
will have the X-Robots-Tag response header removed, allowing
trusted crawlers to index the site.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

1 files changed, 8 insertions, 0 deletions

diff --git a/Caddyfile b/Caddyfile
index fa8e706..7e05478 100644
--- a/Caddyfile
+++ b/Caddyfile
@@ -1,12 +1,20 @@
 golive.yyamashita.com {
     header Referrer-Policy "no-referrer"
     header X-Robots-Tag "noindex, nofollow"
+
+    @crawler_bypass header X-Crawler-Access "f9a3c1e847b2d056"
+    header @crawler_bypass -X-Robots-Tag
+
     reverse_proxy tokyo-app:3000
 }
 
 whoisband.yyamashita.com {
     header Referrer-Policy "no-referrer"
     header X-Robots-Tag "noindex, nofollow"
+
+    @crawler_bypass header X-Crawler-Access "f9a3c1e847b2d056"
+    header @crawler_bypass -X-Robots-Tag
+
     reverse_proxy whois-app:3000
 }
 code.yyamashita.com {