MAINT: Exclude devdocs from search engine results, except devdocs/devel/

j1642 · web-flow · commit 16a934b3e9e1 · 2023-07-05T10:50:37.000-04:00
diff --git a/robots.txt b/robots.txt
@@ -6,7 +6,6 @@ User-agent: *
 
 # do not search root directory by default.
 Disallow: /api/
-Disallow: /devdocs/
 Disallow: /devel/
 Disallow: /examples/
 Disallow: /faq/
@@ -39,6 +38,22 @@ Disallow: /py-modindex.html
 Disallow: /search.html
 Disallow: /searchindex.js
 Disallow: /win32_*.tar.gz
+# exclude all of /devdocs except /devdocs/devel
+Disallow: /devdocs/api/
+Disallow: /devdocs/faq/
+Disallow: /devdocs/gallery/
+Disallow: /devdocs/plot_types/
+Disallow: /devdocs/resources/
+Disallow: /devdocs/thirdpartypackages/
+Disallow: /devdocs/tutorials/
+Disallow: /devdocs/users/
+Disallow: /devdocs/citing.html
+Disallow: /devdocs/contents.html
+Disallow: /devdocs/index.html
+Disallow: /devdocs/objects.inv
+Disallow: /devdocs/py-modindex.html
+Disallow: /devdocs/search.html
+Disallow: /devdocs/searchindex.js
 
 # tell robots this is sitemap
 Sitemap: https://matplotlib.org/_sitemap/sitemap.xml
