feat: Disallow all web crawlers (#111)

* feat: Disallow all web crawlers

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

api/__init__.py

@@ -9,7 +9,17 @@ from api.compat import github as compat
 from api.donations import donations
 from api.announcements import announcements
 from api.login import login
+from api.robots import robots
 
 api = Blueprint.group(
-    login, ping, github, info, socials, donations, announcements, compat, url_prefix="/"
+    login,
+    ping,
+    github,
+    info,
+    socials,
+    donations,
+    announcements,
+    compat,
+    robots,
+    url_prefix="/",
 )

api/robots.py

@@ -0,0 +1,10 @@
+from sanic import Blueprint
+from sanic.response import text
+
+
+robots: Blueprint = Blueprint("robots")
+
+
+@robots.get("/robots.txt")
+async def robots_txt(request):
+    return text("User-agent: *\nDisallow: /", content_type="text/plain")