diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py index 9d56ee9d57..fae3e521b8 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py @@ -36,8 +36,6 @@ EXCLUDED_URLS = [ "https://giphy.com", "https://giphy.com/apps/giphycapture", "https://www.udemy.com/course/the-complete-react-native-and-redux-course/", - # Temporarily unavailable - "https://zulip.com/accounts/find/", ] VNU_IGNORE = [ @@ -150,10 +148,12 @@ class BaseDocumentationSpider(scrapy.Spider): if split_url.netloc == "localhost:9981" and split_url.path in ["", "/"]: return - # This page has some invisible to the user anchor links like #all + # These pages have some invisible to the user anchor links like #all # that are currently invisible, and thus would otherwise fail this test. if url.startswith("http://localhost:9981/communities"): return + if url.startswith("http://localhost:9981/plans"): + return callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse dont_filter = False