From 2bf5139d2368b691fe866f13333ff258c3912ee1 Mon Sep 17 00:00:00 2001 From: Tim Abbott Date: Thu, 9 May 2024 11:27:36 -0700 Subject: [PATCH] tools: Update some exclusion rules. --- .../documentation_crawler/spiders/common/spiders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py index 9d56ee9d57..fae3e521b8 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py @@ -36,8 +36,6 @@ EXCLUDED_URLS = [ "https://giphy.com", "https://giphy.com/apps/giphycapture", "https://www.udemy.com/course/the-complete-react-native-and-redux-course/", - # Temporarily unavailable - "https://zulip.com/accounts/find/", ] VNU_IGNORE = [ @@ -150,10 +148,12 @@ class BaseDocumentationSpider(scrapy.Spider): if split_url.netloc == "localhost:9981" and split_url.path in ["", "/"]: return - # This page has some invisible to the user anchor links like #all + # These pages have some invisible to the user anchor links like #all # that are currently invisible, and thus would otherwise fail this test. if url.startswith("http://localhost:9981/communities"): return + if url.startswith("http://localhost:9981/plans"): + return callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse dont_filter = False