Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

tools: Update some exclusion rules.

This commit is contained in:
Tim Abbott 2024-05-09 11:27:36 -07:00
parent 08c916b337
commit 2bf5139d23
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
View File

@ -36,8 +36,6 @@ EXCLUDED_URLS = [
"https://giphy.com", "https://giphy.com",
"https://giphy.com/apps/giphycapture", "https://giphy.com/apps/giphycapture",
"https://www.udemy.com/course/the-complete-react-native-and-redux-course/", "https://www.udemy.com/course/the-complete-react-native-and-redux-course/",
# Temporarily unavailable
"https://zulip.com/accounts/find/",
] ]
VNU_IGNORE = [ VNU_IGNORE = [
@ -150,10 +148,12 @@ class BaseDocumentationSpider(scrapy.Spider):
if split_url.netloc == "localhost:9981" and split_url.path in ["", "/"]: if split_url.netloc == "localhost:9981" and split_url.path in ["", "/"]:
return return
# This page has some invisible to the user anchor links like #all # These pages have some invisible to the user anchor links like #all
# that are currently invisible, and thus would otherwise fail this test. # that are currently invisible, and thus would otherwise fail this test.
if url.startswith("http://localhost:9981/communities"): if url.startswith("http://localhost:9981/communities"):
return return
if url.startswith("http://localhost:9981/plans"):
return
callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse callback: Callable[[Response], Optional[Iterator[Request]]] = self.parse
dont_filter = False dont_filter = False