Merge pull request clue#42 from clue-labs/robots

Add `robots.txt`
clue-labs · Dec 9, 2024 · 7e04181 · 7e04181
2 parents c868fb2 + 859b6db
commit 7e04181
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -6,7 +6,7 @@ build: public/src/tailwind.min.css
 	cp overrides/* source/overrides/
 	docker run --rm -i -v ${PWD}/source:/docs -u $(shell id -u) squidfunk/mkdocs-material:8.1.3 build
 	cp -r source/build/docs/ build/ && rm build/docs/sitemap.xml.gz
-	cp public/.htaccess public/index.html build/
+	cp public/.htaccess public/index.html public/robots.txt build/
 	cp public/src/* build/src/
 
 public/src/tailwind.min.css: public/index.html tailwindcss

diff --git a/public/robots.txt b/public/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow:
diff --git a/tests/integration.bash b/tests/integration.bash
@@ -33,6 +33,10 @@ match -iP "Content-Type: text/html[\r\n]"
 match -iP "Content-Encoding: gzip[\r\n]"
 match -iP "Vary: Accept-Encoding[\r\n]"
 
+curl -v $base/robots.txt
+match "HTTP/.* 200"
+match -iP "Content-Type: text/plain[;\r\n]"
+
 curl -v $base/invalid
 match "HTTP/.* 404"
 match -i "Content-Type: text/html"