From 6701c4463ccd6df62f9a4c1fab615d707ad93408 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 28 Aug 2019 02:58:53 -0700 Subject: [PATCH] search: Remove now unnecessary tsearch_extra dependency. Now that we're implemented tsearch_extras in pure postgres, we no longer need a custom extension. This should help us considerably, as it means we no longer need to ship custom apt packages at all. Fixes #467. Signed-off-by: Anders Kaseorg --- Dockerfile-postgresql | 24 ++----------------- docs/development/setup-advanced.md | 6 ----- docs/overview/architecture-overview.md | 3 +-- docs/subsystems/full-text-search.md | 7 ------ docs/testing/testing.md | 14 +---------- puppet/zulip/manifests/postgres_appdb_base.pp | 22 ----------------- puppet/zulip/manifests/postgres_common.pp | 3 --- scripts/lib/build-tsearch-extras | 14 ----------- scripts/setup/create-db.sql | 1 - tools/lib/provision.py | 23 ++++-------------- tools/setup/postgres-init-dev-db | 1 - version.py | 2 +- 12 files changed, 9 insertions(+), 111 deletions(-) delete mode 100755 scripts/lib/build-tsearch-extras diff --git a/Dockerfile-postgresql b/Dockerfile-postgresql index c286ea22b0..044fcdf756 100644 --- a/Dockerfile-postgresql +++ b/Dockerfile-postgresql @@ -1,31 +1,13 @@ # To build run `docker build -f Dockerfile-postgresql .` from the root of the # zulip repo. -# Install build tools and build tsearch_extras for the current postgres -# version. Currently the postgres images do not support automatic upgrading of +# Currently the postgres images do not support automatic upgrading of # the on-disk data in volumes. So the base image can not currently be upgraded # without users needing a manual pgdump and restore. -FROM postgres:10 -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - postgresql-server-dev-$PG_MAJOR \ - postgresql-server-dev-all \ - git \ - build-essential \ - fakeroot \ - devscripts -RUN git clone https://github.com/zulip/tsearch_extras.git \ - && cd tsearch_extras \ - && echo $PG_MAJOR > debian/pgversions \ - && pg_buildext updatecontrol \ - && debuild -b -uc -us -# Install tsearch_extras, hunspell, zulip stop words, and run zulip database +# Install hunspell, zulip stop words, and run zulip database # init. FROM postgres:10 -ENV TSEARCH_EXTRAS_VERSION=0.4 -ENV TSEARCH_EXTRAS_DEB=postgresql-${PG_MAJOR}-tsearch-extras_${TSEARCH_EXTRAS_VERSION}_amd64.deb -COPY --from=0 /${TSEARCH_EXTRAS_DEB} /tmp COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/$PG_MAJOR/tsearch_data/zulip_english.stop COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql @@ -36,8 +18,6 @@ RUN apt-key add /tmp/pgroonga-debian.asc \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ hunspell-en-us \ postgresql-${PG_MAJOR}-pgroonga \ - && DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/${TSEARCH_EXTRAS_DEB} \ - && rm /tmp/${TSEARCH_EXTRAS_DEB} \ && ln -sf /var/cache/postgresql/dicts/en_us.dict "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.dict" \ && ln -sf /var/cache/postgresql/dicts/en_us.affix "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.affix" \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/development/setup-advanced.md b/docs/development/setup-advanced.md index a1b24a985b..7acf736d56 100644 --- a/docs/development/setup-advanced.md +++ b/docs/development/setup-advanced.md @@ -101,12 +101,6 @@ git remote add -f upstream https://github.com/zulip/zulip.git doas pkg_add sudo bash gcc postgresql-server redis rabbitmq \ memcached libmemcached py-Pillow py-cryptography py-cffi -# Get tsearch_extras and build it (using a modified version which -# aliases int4 on OpenBSD): -git clone https://github.com/blablacio/tsearch_extras -cd tsearch_extras -gmake && sudo gmake install - # Point environment to custom include locations and use newer GCC # (needed for Node modules): export CFLAGS="-I/usr/local/include -I/usr/local/include/sasl" diff --git a/docs/overview/architecture-overview.md b/docs/overview/architecture-overview.md index bce95d3922..98fea1dae1 100644 --- a/docs/overview/architecture-overview.md +++ b/docs/overview/architecture-overview.md @@ -27,8 +27,7 @@ adapter](https://github.com/zulip/hubot-zulip); integrations with [Jenkins](https://github.com/zulip/zulip-jenkins-plugin), [Puppet](https://github.com/matthewbarr/puppet-zulip), [Redmine](https://github.com/zulip/zulip-redmine-plugin), and -[Trello](https://github.com/zulip/trello-to-zulip); our [full-text -search PostgreSQL extension](https://github.com/zulip/tsearch_extras); +[Trello](https://github.com/zulip/trello-to-zulip); and [many more](https://github.com/zulip/). We use [Transifex](https://www.transifex.com/zulip/zulip/) to do diff --git a/docs/subsystems/full-text-search.md b/docs/subsystems/full-text-search.md index 44626ee162..bd3dd02caa 100644 --- a/docs/subsystems/full-text-search.md +++ b/docs/subsystems/full-text-search.md @@ -17,13 +17,6 @@ feature](http://www.postgresql.org/docs/current/static/textsearch.html), with a custom set of English stop words to improve the quality of the search results. -We use a small extension, -[tsearch_extras](https://github.com/zulip/tsearch_extras), for -highlighting of the matching words. There is [some discussion of -removing this extension, at least as an -option](https://github.com/zulip/zulip/issues/467), so that Zulip can -be used with database-as-a-service platforms. - In order to optimize the performance of delivering messages, the full-text search index is updated for newly sent messages in the background, after the message has been delivered. This background diff --git a/docs/testing/testing.md b/docs/testing/testing.md index 1d1d920f3d..eda7aa9ce4 100644 --- a/docs/testing/testing.md +++ b/docs/testing/testing.md @@ -102,19 +102,7 @@ something valuable to helping keep Zulip bug-free. ### Possible testing issues -- When running the test suite, if you get an error like this: - - ``` - sqlalchemy.exc.ProgrammingError: (ProgrammingError) function ts_match_locs_array(unknown, text, tsquery) does not exist - LINE 2: ...ECT message_id, flags, subject, rendered_content, ts_match_l... - ^ - ``` - - … then you need to install tsearch-extras, described - above. Afterwards, re-run the `init*-db` and the - `do-destroy-rebuild*-database` scripts. - -- Or, when running the test suite, if you get an error involving Git that looks like this: +- When running the test suite, if you get an error involving Git that looks like this: ``` gitlint| An error occurred while executing '/usr/bin/git rev-list --max-count=-1 upstream/master..HEAD': b"fatal: ambiguous argument 'upstream/master..HEAD': unknown revision or path not in the working tree.\nUse '--' to separate paths from revisions, like this:\n'git [...] -- [...]'" diff --git a/puppet/zulip/manifests/postgres_appdb_base.pp b/puppet/zulip/manifests/postgres_appdb_base.pp index 146d762c5c..69ef1828be 100644 --- a/puppet/zulip/manifests/postgres_appdb_base.pp +++ b/puppet/zulip/manifests/postgres_appdb_base.pp @@ -8,24 +8,6 @@ class zulip::postgres_appdb_base { 'debian': { include zulip::apt_repository $postgresql = "postgresql-${zulip::base::postgres_version}" - $appdb_packages = [ - # Needed for our full text search system - "${postgresql}-tsearch-extras", - ] - if $zulip::base::release_name == 'stretch' { - zulip::safepackage { - $appdb_packages: - ensure => 'installed', - require => Exec['setup_apt_repo'], - } - } - else { - exec {'build_tsearch_extras': - require => Package[$zulip::postgres_common::postgresql_dev], - command => "bash -c ${::zulip_scripts_path}/lib/build-tsearch-extras", - creates => "/usr/pgsql-${zulip::base::postgres_version}/lib/tsearch_extras.so", - } - } $postgres_sharedir = "/usr/share/postgresql/${zulip::base::postgres_version}" $tsearch_datadir = "${postgres_sharedir}/tsearch_data" $pgroonga_setup_sql_path = "${postgres_sharedir}/pgroonga_setup.sql" @@ -34,10 +16,6 @@ class zulip::postgres_appdb_base { 'redhat': { include zulip::yum_repository $postgresql = "postgresql${zulip::base::postgres_version}" - exec {'build_tsearch_extras': - command => "bash -c ${::zulip_scripts_path}/lib/build-tsearch-extras", - creates => "/usr/pgsql-${zulip::base::postgres_version}/lib/tsearch_extras.so", - } $postgres_sharedir = "/usr/pgsql-${zulip::base::postgres_version}/share" $tsearch_datadir = "${postgres_sharedir}/tsearch_data/" $pgroonga_setup_sql_path = "${postgres_sharedir}/pgroonga_setup.sql" diff --git a/puppet/zulip/manifests/postgres_common.pp b/puppet/zulip/manifests/postgres_common.pp index 75f2bf63b8..17925a301b 100644 --- a/puppet/zulip/manifests/postgres_common.pp +++ b/puppet/zulip/manifests/postgres_common.pp @@ -3,12 +3,9 @@ class zulip::postgres_common { case $::osfamily { 'debian': { $postgresql = "postgresql-${zulip::base::postgres_version}" - $postgresql_dev = "postgresql-server-dev-${zulip::base::postgres_version}" $postgres_packages = [ # The database itself $postgresql, - # Developer pacakge for building extensions (such as tsearch_extras) - $postgresql_dev, # tools for database monitoring; formerly ptop 'pgtop', # Needed just to support adding postgres user to 'zulip' group diff --git a/scripts/lib/build-tsearch-extras b/scripts/lib/build-tsearch-extras deleted file mode 100755 index bf74f575f9..0000000000 --- a/scripts/lib/build-tsearch-extras +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -x -set -e - -cd "$(mktemp -d)" -distro=$(lsb_release -is) - -git clone https://github.com/zulip/tsearch_extras -if [ "$distro" = "Ubuntu" ] || [ "$distro" == "Debian" ]; then - make="make" -else - make="gmake" -fi -(cd tsearch_extras && "$make" && "$make" install) diff --git a/scripts/setup/create-db.sql b/scripts/setup/create-db.sql index 52135adaf8..0a714db218 100644 --- a/scripts/setup/create-db.sql +++ b/scripts/setup/create-db.sql @@ -3,4 +3,3 @@ ALTER ROLE zulip SET search_path TO zulip,public; CREATE DATABASE zulip OWNER=zulip; \connect zulip CREATE SCHEMA zulip AUTHORIZATION zulip; -CREATE EXTENSION tsearch_extras SCHEMA zulip; diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 13d4db7e51..2687d1844a 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -171,32 +171,23 @@ COMMON_YUM_DEPENDENCIES = COMMON_DEPENDENCIES + [ "libstdc++" ] + YUM_THUMBOR_VENV_DEPENDENCIES -BUILD_TSEARCH_FROM_SOURCE = False BUILD_PGROONGA_FROM_SOURCE = False if vendor == "ubuntu" and os_version in ("18.10", "19.04"): - # For platforms without a tsearch-extras package distributed - # from our PPA, we need to build from source. - BUILD_TSEARCH_FROM_SOURCE = True SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ pkg.format(POSTGRES_VERSION) for pkg in [ "postgresql-{0}", "postgresql-{0}-pgroonga", - # Dependency for building tsearch_extras from source - "postgresql-server-dev-{0}", ] ] elif vendor == 'debian' and os_version == "10": - # For platforms without a tsearch-extras package distributed - # from our PPA or a pgroonga release, we need to build both + # For platforms without a pgroonga release, we need to build it # from source. BUILD_PGROONGA_FROM_SOURCE = True - BUILD_TSEARCH_FROM_SOURCE = True SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [ pkg.format(POSTGRES_VERSION) for pkg in [ "postgresql-{0}", - # Dependency for building tsearch_extras from source - "postgresql-server-dev-{0}", # Dependency for building pgroonga from source + "postgresql-server-dev-{0}", "libgroonga-dev", "libmsgpack-dev", ] @@ -206,7 +197,6 @@ elif vendor in ["ubuntu", "debian"]: pkg.format(POSTGRES_VERSION) for pkg in [ "postgresql-{0}", "postgresql-{0}-pgroonga", - "postgresql-{0}-tsearch-extras", ] ] elif vendor in ["CentOS", "RedHat"]: @@ -214,11 +204,9 @@ elif vendor in ["CentOS", "RedHat"]: pkg.format(POSTGRES_VERSION) for pkg in [ "postgresql{0}-server", "postgresql{0}", - "postgresql{0}-devel", "postgresql{0}-pgroonga", ] ] + REDHAT_VENV_DEPENDENCIES - BUILD_TSEARCH_FROM_SOURCE = True elif vendor == "Fedora": SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [ pkg.format(POSTGRES_VERSION) for pkg in [ @@ -230,7 +218,6 @@ elif vendor == "Fedora": "msgpack-devel", ] ] + FEDORA_VENV_DEPENDENCIES - BUILD_TSEARCH_FROM_SOURCE = True BUILD_PGROONGA_FROM_SOURCE = True if family == 'redhat': @@ -259,12 +246,10 @@ def install_system_deps(): else: raise AssertionError("Invalid vendor") - # For some platforms, there aren't published pgroonga or - # tsearch-extra packages available, so we build them from source. + # For some platforms, there aren't published pgroonga + # packages available, so we build them from source. if BUILD_PGROONGA_FROM_SOURCE: run_as_root(["./scripts/lib/build-pgroonga"]) - if BUILD_TSEARCH_FROM_SOURCE: - run_as_root(["./scripts/lib/build-tsearch-extras"]) def install_apt_deps(deps_to_install): # type: (List[str]) -> None diff --git a/tools/setup/postgres-init-dev-db b/tools/setup/postgres-init-dev-db index cfc69685d1..1843bf4e65 100755 --- a/tools/setup/postgres-init-dev-db +++ b/tools/setup/postgres-init-dev-db @@ -98,7 +98,6 @@ CREATE SCHEMA zulip; EOF "${ROOT_POSTGRES[@]}" -v ON_ERROR_STOP=1 -e "$DBNAME_BASE" << EOF -CREATE EXTENSION tsearch_extras SCHEMA zulip; CREATE EXTENSION pgroonga; GRANT USAGE ON SCHEMA pgroonga TO $USERNAME; EOF diff --git a/version.py b/version.py index fb40c245be..bce847d1a2 100644 --- a/version.py +++ b/version.py @@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea # historical commits sharing the same major version, in which case a # minor version bump suffices. -PROVISION_VERSION = '51.0' +PROVISION_VERSION = '52.0'