From 7c023042cfbbd8817d5ba3adfca365734500a0e2 Mon Sep 17 00:00:00 2001
From: Alex Vandiver <alexmv@zulip.com>
Date: Thu, 28 Apr 2022 16:23:27 -0700
Subject: [PATCH] puppet: Rotate access log files every day, not at 500M.

Since logrotate runs in a daily cron, this practically means "daily,
but only if it's larger than 500M."  For large installs with large
traffic, this is effectively daily for 10 days; for small installs, it
is an unknown amount of time.

Switch to daily logfiles, defaulting to 14 days to match nginx; this
can be overridden using a zulip.conf setting.  This makes it easier to
ensure that access logs are only kept for a bounded period of time.
---
 docs/production/deployment.md                   |  5 +++++
 puppet/zulip/manifests/nginx.pp                 |  3 ++-
 puppet/zulip/manifests/profile/app_frontend.pp  | 11 ++++++-----
 .../logrotate/nginx.template.erb}               |  2 +-
 .../logrotate/zulip.template.erb}               |  4 ++--
 scripts/log-search                              | 17 +++++++++++++++--
 6 files changed, 31 insertions(+), 11 deletions(-)
 rename puppet/zulip/{files/logrotate/nginx => templates/logrotate/nginx.template.erb} (88%)
 rename puppet/zulip/{files/logrotate/zulip => templates/logrotate/zulip.template.erb} (94%)

diff --git a/docs/production/deployment.md b/docs/production/deployment.md
index f761c03a06..a036990c02 100644
--- a/docs/production/deployment.md
+++ b/docs/production/deployment.md
@@ -791,6 +791,11 @@ Override the default uwsgi backlog of 128 connections.
 Override the default `uwsgi` (Django) process count of 6 on hosts with
 more than 3.5GiB of RAM, 4 on hosts with less.
 
+#### `access_log_retention_days`
+
+Number of days of access logs to keep, for both nginx and the application.
+Defaults to 14 days.
+
 ### `[postfix]`
 
 #### `mailname`
diff --git a/puppet/zulip/manifests/nginx.pp b/puppet/zulip/manifests/nginx.pp
index d2743fb2f6..d6115c8cf1 100644
--- a/puppet/zulip/manifests/nginx.pp
+++ b/puppet/zulip/manifests/nginx.pp
@@ -76,13 +76,14 @@ class zulip::nginx {
     group  => 'adm',
     mode   => '0750',
   }
+  $access_log_retention_days = zulipconf('application_server','access_log_retention_days', 14)
   file { '/etc/logrotate.d/nginx':
     ensure  => file,
     require => Package[$zulip::common::nginx],
     owner   => 'root',
     group   => 'root',
     mode    => '0644',
-    source  => 'puppet:///modules/zulip/logrotate/nginx',
+    content => template('zulip/logrotate/nginx.template.erb'),
   }
   package { 'certbot':
     ensure => installed,
diff --git a/puppet/zulip/manifests/profile/app_frontend.pp b/puppet/zulip/manifests/profile/app_frontend.pp
index 42242a4fff..e82ff71d0c 100644
--- a/puppet/zulip/manifests/profile/app_frontend.pp
+++ b/puppet/zulip/manifests/profile/app_frontend.pp
@@ -23,12 +23,13 @@ class zulip::profile::app_frontend {
     content => template('zulip/nginx/zulip-enterprise.template.erb'),
     notify  => Service['nginx'],
   }
+  $access_log_retention_days = zulipconf('application_server','access_log_retention_days', 14)
   file { '/etc/logrotate.d/zulip':
-    ensure => file,
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0644',
-    source => 'puppet:///modules/zulip/logrotate/zulip',
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('zulip/logrotate/zulip.template.erb'),
   }
   file { '/etc/nginx/sites-enabled/zulip-enterprise':
     ensure  => link,
diff --git a/puppet/zulip/files/logrotate/nginx b/puppet/zulip/templates/logrotate/nginx.template.erb
similarity index 88%
rename from puppet/zulip/files/logrotate/nginx
rename to puppet/zulip/templates/logrotate/nginx.template.erb
index ad1d97af54..052955a529 100644
--- a/puppet/zulip/files/logrotate/nginx
+++ b/puppet/zulip/templates/logrotate/nginx.template.erb
@@ -1,7 +1,7 @@
 /var/log/nginx/*.log {
 	daily
 	missingok
-	rotate 14
+	rotate <%= @access_log_retention_days %>
 	compress
 	delaycompress
 	notifempty
diff --git a/puppet/zulip/files/logrotate/zulip b/puppet/zulip/templates/logrotate/zulip.template.erb
similarity index 94%
rename from puppet/zulip/files/logrotate/zulip
rename to puppet/zulip/templates/logrotate/zulip.template.erb
index 2858ef1b50..0248291cca 100644
--- a/puppet/zulip/files/logrotate/zulip
+++ b/puppet/zulip/templates/logrotate/zulip.template.erb
@@ -32,8 +32,8 @@
 /var/log/zulip/server.log
 {
     missingok
-    rotate 10
-    size 500M
+    rotate <%= @access_log_retention_days %>
+    daily
     compress
     delaycompress
     notifempty
diff --git a/scripts/log-search b/scripts/log-search
index 2e8297f018..745b23ee23 100755
--- a/scripts/log-search
+++ b/scripts/log-search
@@ -25,18 +25,31 @@ from typing import Protocol
 
 from django.conf import settings
 
-from scripts.lib.zulip_tools import BOLD, CYAN, ENDC, FAIL, GRAY, OKBLUE
+from scripts.lib.zulip_tools import (
+    BOLD,
+    CYAN,
+    ENDC,
+    FAIL,
+    GRAY,
+    OKBLUE,
+    get_config,
+    get_config_file,
+)
 
 
 def parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description="Search logfiles, ignoring commonly-fetched URLs.")
     log_selection = parser.add_argument_group("File selection")
     log_selection_options = log_selection.add_mutually_exclusive_group()
+    access_log_retention_days = int(
+        get_config(get_config_file(), "application_server", "access_log_retention_days", "14")
+    )
     log_selection_options.add_argument(
         "--log-files",
         "-n",
         help="Number of log files to search",
-        choices=range(1, 16),
+        choices=range(1, access_log_retention_days + 2),
+        metavar=f"[1-{access_log_retention_days+1}]",
         type=int,
     )
     log_selection_options.add_argument(