MAINT: Exclude /devdocs from search engine results, except /devdocs/devel

j1642 · web-flow · commit 04aa27c489d6 · 2023-07-06T15:32:15.000-04:00
diff --git a/robots.txt b/robots.txt
@@ -39,21 +39,8 @@ Disallow: /search.html
 Disallow: /searchindex.js
 Disallow: /win32_*.tar.gz
 # exclude all of /devdocs except /devdocs/devel
-Disallow: /devdocs/api/
-Disallow: /devdocs/faq/
-Disallow: /devdocs/gallery/
-Disallow: /devdocs/plot_types/
-Disallow: /devdocs/resources/
-Disallow: /devdocs/thirdpartypackages/
-Disallow: /devdocs/tutorials/
-Disallow: /devdocs/users/
-Disallow: /devdocs/citing.html
-Disallow: /devdocs/contents.html
-Disallow: /devdocs/index.html
-Disallow: /devdocs/objects.inv
-Disallow: /devdocs/py-modindex.html
-Disallow: /devdocs/search.html
-Disallow: /devdocs/searchindex.js
+Disallow: /devdocs/
+Allow: /devdocs/devel/
 
 # tell robots this is sitemap
 Sitemap: https://matplotlib.org/_sitemap/sitemap.xml
