blender_extensions/urls.py

@@ -18,7 +18,7 @@ from django.contrib import admin
 from django.contrib.flatpages import views as flatpages_views
 from django.urls import path, include, re_path
 from django.views.static import serve
-from django.views.generic.base import RedirectView
+from django.views.generic.base import RedirectView, TemplateView
 
 from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
 
@@ -44,6 +44,7 @@ urlpatterns = [
     path('api/swagger/', RedirectView.as_view(url='/api/v1/swagger/')),
     path('api/v1/', SpectacularAPIView.as_view(), name='schema_v1'),
     path('api/v1/swagger/', SpectacularSwaggerView.as_view(url_name='schema_v1'), name='swagger'),
+    path('robots.txt', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),
     # Flatpages
     path('about/', flatpages_views.flatpage, {'url': '/about/'}, name='flatpage-about'),
     re_path(r'^(?P<url>.*/)$', flatpages_views.flatpage),

extensions/templates/robots.txt

@@ -0,0 +1,5 @@
+User-Agent: *
+Disallow: /approval-queue/
+
+User-agent: GPTBot
+Disallow: /

extensions/tests/test_views.py

@@ -309,3 +309,11 @@ class MyExtensionsTest(_BaseTestCase):
         self.client.force_login(user)
         response = self.client.get(reverse('extensions:manage-list'))
         self.assertContains(response, extension.name)
+
+
+class RobotsTxtTests(_BaseTestCase):
+    def test_get(self):
+        response = self.client.get("/robots.txt")
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response["content-type"], "text/plain")