Mirror
/
zulip
mirror of https://github.com/zulip/zulip.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

tests: Remove slow tests detection. 

According to @showell:

> All the slow decorators can die. That was a failed experiment of
> mine from 2014 days.  I have meaning to kill them for a couple years
> now.  I wrote this with the best of intentions, but I believe it's
> now just cruft.  We never made a "fast" mode, for one. And we kept
> writing more and more slow tests, haha.

Signed-off-by: Anders Kaseorg <anders@zulip.com>
This commit is contained in:
Anders Kaseorg 2020-06-30 19:29:42 -07:00
parent 89ab3682f1
commit 92d0290dc5
13 changed files with 0 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tools/test-backend
View File

@ -241,10 +241,6 @@ def main() -> None:
action="store_true",
default=False,
help="Show detailed output")
parser.add_argument('--report-slow-tests', dest='report_slow_tests',
action="store_true",
default=False,
help="Show which tests are slowest.")
parser.add_argument('--reverse', dest='reverse',
action="store_true",
default=False,
@ -458,13 +454,6 @@ def main() -> None:
print("1.) `vagrant ssh -- -L 8080:127.0.0.1:8080`")
print(f"2.) `snakeviz -s {shlex.quote(stats_file.name)}`")
if options.report_slow_tests:
from zerver.lib.test_runner import report_slow_tests
# We do this even with failures, since slowness can be
# an important clue as to why tests fail.
report_slow_tests()
# Ideally, we'd check for any leaked test databases here;
# but that needs some hackery with database names.
#

69
zerver/lib/test_runner.py
View File

@ -2,7 +2,6 @@ import os
import random
import shutil
import sys
import time
import unittest
from functools import partial
from multiprocessing.sharedctypes import Synchronized
@ -49,75 +48,14 @@ _worker_id = 0 # Used to identify the worker process.
ReturnT = TypeVar('ReturnT') # Constrain return type to match
def slow(slowness_reason: str) -> Callable[[Callable[..., ReturnT]], Callable[..., ReturnT]]:
'''
This is a decorate that annotates a test as being "known
to be slow." The decorator will set expected_run_time and slowness_reason
as attributes of the function. Other code can use this annotation
as needed, e.g. to exclude these tests in "fast" mode.
'''
def decorator(f: Callable[..., ReturnT]) -> Callable[..., ReturnT]:
setattr(f, 'slowness_reason', slowness_reason)
return f
return decorator
def is_known_slow_test(test_method: Callable[..., ReturnT]) -> bool:
return hasattr(test_method, 'slowness_reason')
def full_test_name(test: TestCase) -> str:
test_module = test.__module__
test_class = test.__class__.__name__
test_method = test._testMethodName
return f'{test_module}.{test_class}.{test_method}'
def get_test_method(test: TestCase) -> Callable[[], None]:
return getattr(test, test._testMethodName)
# Each tuple is delay, test_name, slowness_reason
TEST_TIMINGS: List[Tuple[float, str, str]] = []
def report_slow_tests() -> None:
timings = sorted(TEST_TIMINGS, reverse=True)
print('SLOWNESS REPORT')
print(' delay test')
print(' ---- ----')
for delay, test_name, slowness_reason in timings[:15]:
if not slowness_reason:
slowness_reason = 'UNKNOWN WHY SLOW, please investigate'
print(f' {delay:0.3f} {test_name}\n {slowness_reason}\n')
print('...')
for delay, test_name, slowness_reason in timings[100:]:
if slowness_reason:
print(f' {delay:.3f} {test_name} is not that slow')
print(' consider removing @slow decorator')
print(f' This may no longer be true: {slowness_reason}')
def enforce_timely_test_completion(test_method: Callable[..., ReturnT], test_name: str,
delay: float, result: unittest.TestResult) -> None:
if hasattr(test_method, 'slowness_reason'):
max_delay = 2.0 # seconds
else:
max_delay = 0.4 # seconds
assert isinstance(result, (TextTestResult, RemoteTestResult))
if delay > max_delay:
msg = f'** Test is TOO slow: {test_name} ({delay:.3f} s)\n'
result.addInfo(test_method, msg)
def fast_tests_only() -> bool:
return "FAST_TESTS_ONLY" in os.environ
def run_test(test: TestCase, result: TestResult) -> bool:
failed = False
test_method = get_test_method(test)
if fast_tests_only() and is_known_slow_test(test_method):
return failed
test_name = full_test_name(test)
bounce_key_prefix_for_testing(test_name)
@ -129,15 +67,8 @@ def run_test(test: TestCase, result: TestResult) -> bool:
result.addError(test, sys.exc_info())
return True
start_time = time.time()
test(result) # unittest will handle skipping, error, failure and success.
delay = time.time() - start_time
enforce_timely_test_completion(test_method, test_name, delay, result)
slowness_reason = getattr(test_method, 'slowness_reason', '')
TEST_TIMINGS.append((delay, test_name, slowness_reason))
test._post_teardown()
return failed

4
zerver/tests/test_docs.py
View File

@ -11,7 +11,6 @@ from django.test import TestCase, override_settings
from zerver.lib.integrations import INTEGRATIONS
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_helpers import HostRequestMock
from zerver.lib.test_runner import slow
from zerver.lib.utils import split_by
from zerver.models import Realm, get_realm
from zerver.views.documentation import add_api_uri_context
@ -89,7 +88,6 @@ class DocPageTest(ZulipTestCase):
if not doc_html_str:
self.assert_in_success_response(['<meta name="robots" content="noindex,nofollow">'], result)
@slow("Tests dozens of endpoints")
def test_api_doc_endpoints(self) -> None:
current_dir = os.path.dirname(os.path.abspath(__file__))
api_docs_dir = os.path.join(current_dir, '..', '..', 'templates/zerver/api/')
@ -105,7 +103,6 @@ class DocPageTest(ZulipTestCase):
endpoint = f'/api/{os.path.splitext(f)[0]}'
self._test(endpoint, '', doc_html_str=True)
@slow("Tests dozens of endpoints, including generating lots of emails")
def test_doc_endpoints(self) -> None:
self._test('/api/', 'The Zulip API')
self._test('/api/api-keys', 'be careful with it')
@ -167,7 +164,6 @@ class DocPageTest(ZulipTestCase):
self._test(url, title, doc_html_str=True)
self._test(url, description, doc_html_str=True)
@slow("Tests dozens of endpoints, including all our integrations docs")
def test_integration_doc_endpoints(self) -> None:
self._test('/integrations/',
'native integrations.',

8
zerver/tests/test_events.py
View File

@ -102,7 +102,6 @@ from zerver.lib.test_helpers import (
reset_emails_in_zulip_realm,
stdout_suppressed,
)
from zerver.lib.test_runner import slow
from zerver.lib.topic import ORIG_TOPIC, TOPIC_LINKS, TOPIC_NAME
from zerver.lib.validator import (
Validator,
@ -1330,7 +1329,6 @@ class NormalActionsTest(BaseAction):
schema_checker('events[0]', events[0])
avatar_schema_checker('events[1]', events[1])
@slow("Runs a large matrix of tests")
def test_change_realm_authentication_methods(self) -> None:
schema_checker = check_events_dict([
('type', equals('realm')),
@ -1432,7 +1430,6 @@ class NormalActionsTest(BaseAction):
value))
schema_checker('events[0]', events[0])
@slow("Runs a matrix of 6 queries to the /home view")
def test_change_realm_message_edit_settings(self) -> None:
schema_checker = check_events_dict([
('type', equals('realm')),
@ -1558,7 +1555,6 @@ class NormalActionsTest(BaseAction):
lambda: do_change_user_role(self.user_profile, role))
schema_checker('events[0]', events[0])
@slow("Actually runs several full-stack fetching tests")
def test_change_notification_settings(self) -> None:
for notification_setting, v in self.user_profile.notification_setting_types.items():
if notification_setting in ["notification_sound", "desktop_icon_count_display"]:
@ -2504,7 +2500,6 @@ class RealmPropertyActionTest(BaseAction):
state_change_expected=state_change_expected)
schema_checker('events[0]', events[0])
@slow("Actually runs several full-stack fetching tests")
def test_change_realm_property(self) -> None:
for prop in Realm.property_types:
with self.settings(SEND_DIGEST_EMAILS=True):
@ -2582,17 +2577,14 @@ class UserDisplayActionTest(BaseAction):
if setting_name == "timezone":
timezone_schema_checker('events[1]', events[1])
@slow("Actually runs several full-stack fetching tests")
def test_set_user_display_settings(self) -> None:
for prop in UserProfile.property_types:
self.do_set_user_display_settings_test(prop)
class SubscribeActionTest(BaseAction):
@slow("Actually several tests combined together")
def test_subscribe_events(self) -> None:
self.do_test_subscribe_events(include_subscribers=True)
@slow("Actually several tests combined together")
def test_subscribe_events_no_include_subscribers(self) -> None:
self.do_test_subscribe_events(include_subscribers=False)

4
zerver/tests/test_home.py
View File

@ -16,7 +16,6 @@ from zerver.lib.events import add_realm_logo_fields
from zerver.lib.soft_deactivation import do_soft_deactivate_users
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_helpers import get_user_messages, queries_captured
from zerver.lib.test_runner import slow
from zerver.lib.users import compute_show_invites_and_add_streams
from zerver.models import (
DefaultStream,
@ -320,7 +319,6 @@ class HomeTest(ZulipTestCase):
self.assert_length(cache_mock.call_args_list, 6)
self.assert_length(queries, 40)
@slow("Creates and subscribes 10 users in a loop. Should use bulk queries.")
def test_num_queries_with_streams(self) -> None:
main_user = self.example_user('hamlet')
other_user = self.example_user('cordelia')
@ -510,7 +508,6 @@ class HomeTest(ZulipTestCase):
page_params = self._get_page_params(result)
self.assertEqual(page_params['realm_signup_notifications_stream_id'], get_stream('Denmark', realm).id)
@slow('creating users and loading home page')
def test_people(self) -> None:
hamlet = self.example_user('hamlet')
realm = get_realm('zulip')
@ -891,7 +888,6 @@ class HomeTest(ZulipTestCase):
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(idle_user_msg_list[-1].content, message)
@slow("Loads home page data several times testing different cases")
def test_multiple_user_soft_deactivations(self) -> None:
long_term_idle_user = self.example_user('hamlet')
# We are sending this message to ensure that long_term_idle_user has

2
zerver/tests/test_import_export.py
View File

@ -24,7 +24,6 @@ from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids
from zerver.lib.streams import create_stream_if_needed
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, use_s3_backend
from zerver.lib.test_runner import slow
from zerver.lib.topic_mutes import add_topic_mute
from zerver.lib.upload import (
claim_attachment,
@ -70,7 +69,6 @@ class QueryUtilTest(ZulipTestCase):
for _ in range(5):
self.send_personal_message(user, self.example_user('othello'))
@slow('creates lots of data')
def test_query_chunker(self) -> None:
self._create_messages()

2
zerver/tests/test_management_commands.py
View File

@ -16,7 +16,6 @@ from zerver.lib.actions import do_add_reaction, do_create_user
from zerver.lib.management import CommandError, ZulipBaseCommand, check_config
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_helpers import most_recent_message, stdout_suppressed
from zerver.lib.test_runner import slow
from zerver.models import (
Message,
Reaction,
@ -188,7 +187,6 @@ class TestCommandsCanStart(TestCase):
if command != '__init__'
]
@slow("Aggregate of runs dozens of individual --help tests")
def test_management_commands_show_help(self) -> None:
with stdout_suppressed():
for command in self.commands:

2
zerver/tests/test_markdown.py
View File

@ -42,7 +42,6 @@ from zerver.lib.mention import possible_mentions, possible_user_group_mentions
from zerver.lib.message import render_markdown
from zerver.lib.request import JsonableError
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_runner import slow
from zerver.lib.tex import render_tex
from zerver.lib.user_groups import create_user_group
from zerver.models import (
@ -389,7 +388,6 @@ class MarkdownTest(ZulipTestCase):
is_ignored = test.get('ignore', False)
self.assertFalse(is_ignored, message)
@slow("Aggregate of runs dozens of individual markdown tests")
def test_markdown_fixtures(self) -> None:
format_tests, linkify_tests = self.load_markdown_tests()
valid_keys = {"name", "input", "expected_output",

4
zerver/tests/test_messages.py
View File

@ -81,7 +81,6 @@ from zerver.lib.test_helpers import (
queries_captured,
reset_emails_in_zulip_realm,
)
from zerver.lib.test_runner import slow
from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp
from zerver.lib.timezone import get_timezone
from zerver.lib.topic import DB_TOPIC_NAME, LEGACY_PREV_TOPIC, TOPIC_LINKS, TOPIC_NAME
@ -364,7 +363,6 @@ class TestCrossRealmPMs(ZulipTestCase):
self.register(email, 'test', subdomain=subdomain)
return get_user(email, get_realm(subdomain))
@slow("Sends a large number of messages")
@override_settings(CROSS_REALM_BOT_EMAILS=['notification-bot@zulip.com',
'welcome-bot@zulip.com',
'support@3.example.com'])
@ -806,7 +804,6 @@ class PersonalMessagesTest(ZulipTestCase):
f'<UserMessage: recip / {user_profile.email} ([])>',
)
@slow("checks several profiles")
def test_personal_to_self(self) -> None:
"""
If you send a personal to yourself, only you see it.
@ -1366,7 +1363,6 @@ class MessageDictTest(ZulipTestCase):
self.assertEqual(send_message_payload, fetch_payload)
@slow('builds lots of messages')
def test_bulk_message_fetching(self) -> None:
sender = self.example_user('othello')
receiver = self.example_user('hamlet')

2
zerver/tests/test_realm.py
View File

@ -22,7 +22,6 @@ from zerver.lib.send_email import send_future_email
from zerver.lib.streams import create_stream_if_needed
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_helpers import reset_emails_in_zulip_realm, tornado_redirected_to_list
from zerver.lib.test_runner import slow
from zerver.models import (
Attachment,
CustomProfileField,
@ -738,7 +737,6 @@ class RealmAPITest(ZulipTestCase):
realm = self.update_with_api(name, vals[0])
self.assertEqual(getattr(realm, name), vals[0])
@slow("Tests a dozen properties in a loop")
def test_update_realm_properties(self) -> None:
for prop in Realm.property_types:
with self.subTest(property=prop):

2
zerver/tests/test_slack_message_conversion.py
View File

@ -9,7 +9,6 @@ from zerver.data_import.slack_message_conversion import (
)
from zerver.lib import mdiff
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_runner import slow
class SlackMessageConversion(ZulipTestCase):
@ -31,7 +30,6 @@ class SlackMessageConversion(ZulipTestCase):
return test_fixtures
@slow("Aggregate of runs of individual slack message conversion tests")
def test_message_conversion_fixtures(self) -> None:
format_tests = self.load_slack_message_conversion_tests()
valid_keys = {'name', "input", "conversion_output"}

4
zerver/tests/test_subs.py
View File

@ -64,7 +64,6 @@ from zerver.lib.test_helpers import (
reset_emails_in_zulip_realm,
tornado_redirected_to_list,
)
from zerver.lib.test_runner import slow
from zerver.models import (
DefaultStream,
DefaultStreamGroup,
@ -3688,7 +3687,6 @@ class InviteOnlyStreamTest(ZulipTestCase):
if sub['name'] == "Saxony":
self.assertEqual(sub['invite_only'], True, "Saxony was not properly marked private")
@slow("lots of queries")
def test_inviteonly(self) -> None:
# Creating an invite-only stream is allowed
hamlet = self.example_user('hamlet')
@ -3786,7 +3784,6 @@ class GetSubscribersTest(ZulipTestCase):
stream_name = gather_subscriptions(self.user_profile)[0][0]['name']
self.make_successful_subscriber_request(stream_name)
@slow("common_subscribe_to_streams is slow")
def test_gather_subscriptions(self) -> None:
"""
gather_subscriptions returns correct results with only 3 queries
@ -3855,7 +3852,6 @@ class GetSubscribersTest(ZulipTestCase):
self.assertTrue(len(sub["subscribers"]) == len(users_to_subscribe))
self.assert_length(queries, 6)
@slow("common_subscribe_to_streams is slow")
def test_never_subscribed_streams(self) -> None:
"""
Check never_subscribed streams are fetched correctly and not include invite_only streams.

2
zerver/tests/test_urls.py
View File

@ -7,7 +7,6 @@ import ujson
from django.test import Client, TestCase
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_runner import slow
from zerver.models import Stream
from zproject import urls
@ -25,7 +24,6 @@ class PublicURLTest(ZulipTestCase):
self.assertEqual(response.status_code, expected_status,
msg=f"Expected {expected_status}, received {response.status_code} for {method} to {url}")
@slow("Tests dozens of endpoints, including all of our /help/ documents")
def test_public_urls(self) -> None:
"""
Test which views are accessible when not logged in.