Debian packaging via dh_virtualenv (#4285)

Author: richvdh

.dockerignore

@@ -5,3 +5,5 @@ demo/etc
 tox.ini
 .git/*
 .tox/*
+debian/matrix-synapse/
+debian/matrix-synapse-*/

.gitignore

@@ -18,14 +18,16 @@ homeserver*.db
 homeserver*.log
 homeserver*.log.*
 homeserver*.pid
-homeserver*.yaml
+/homeserver*.yaml
 
 *.signing.key
 *.tls.crt
 *.tls.dh
 *.tls.key
 
 .coverage
+.coverage.*
+!.coverage.rc
 htmlcov
 
 demo/*/*.db
@@ -57,3 +59,7 @@ env/
 
 .vscode/
 .ropeproject/
+
+*.deb
+
+/debs

MANIFEST.in

@@ -36,6 +36,7 @@ prune demo/etc
 prune docker
 prune .circleci
 prune .coveragerc
+prune debian
 
 exclude jenkins*
 recursive-exclude jenkins *.sh

changelog.d/4212.misc

@@ -0,0 +1 @@
+Debian packages utilising a virtualenv with bundled dependencies can now be built.

debian/.gitignore

@@ -0,0 +1,7 @@
+/matrix-synapse-py3.*.debhelper
+/matrix-synapse-py3.debhelper.log
+/matrix-synapse-py3.substvars
+/matrix-synapse-*/
+/files
+/debhelper-build-stamp
+/.debhelper

debian/NEWS

@@ -0,0 +1,22 @@
+matrix-synapse-py3 (0.34.0) stable; urgency=medium
+
+  matrix-synapse-py3 is intended as a drop-in replacement for the existing
+  matrix-synapse package. The replacement should be relatively seamless,
+  however, please note the following important differences to matrix-synapse:
+
+  * Most importantly, the matrix-synapse service now runs under Python 3 rather
+    than Python 2.7.
+
+  * Synapse is installed into its own virtualenv (in /opt/venvs/matrix-synapse)
+    instead of using the system python libraries. (This may mean that you can
+    remove a number of old dependencies with `apt-get autoremove`).
+
+  matrix-synapse-py3 will take over responsibility for the existing
+  configuration files, including the matrix-synapse systemd service.
+
+  Beware, however, that `apt-get purge matrix-synapse` will *disable* the
+  matrix-synapse service (so that it will not be started on reboot), even
+  though that service is no longer being provided by the matrix-synapse
+  package. It can be re-enabled with `systemctl enable matrix-synapse`.
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 19 Dec 2018 14:00:00 +0000

debian/build_virtualenv

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# runs dh_virtualenv to build the virtualenv in the build directory,
+# and then runs the trial tests against the installed synapse.
+
+set -e
+
+export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
+SNAKE=/usr/bin/python3
+
+# try to set the CFLAGS so any compiled C extensions are compiled with the most
+# generic as possible x64 instructions, so that compiling it on a new Intel chip
+# doesn't enable features not available on older ones or AMD.
+#
+# TODO: add similar things for non-amd64, or figure out a more generic way to
+# do this.
+
+case `dpkg-architecture -q DEB_HOST_ARCH` in
+    amd64)
+        export CFLAGS=-march=x86-64
+        ;;
+esac
+
+# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
+# than the 2/3 compatible `virtualenv`.
+
+dh_virtualenv \
+    --install-suffix "matrix-synapse" \
+    --builtin-venv \
+    --setuptools \
+    --python "$SNAKE" \
+    --upgrade-pip \
+    --preinstall="lxml" \
+    --preinstall="mock" \
+    --extra-pip-arg="--no-cache-dir" \
+    --extra-pip-arg="--compile"
+
+# we copy the tests to a temporary directory so that we can put them on the
+# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+tmpdir=`mktemp -d`
+trap "rm -r $tmpdir" EXIT
+
+cp -r tests "$tmpdir"
+cd debian/matrix-synapse-py3
+
+PYTHONPATH="$tmpdir" \
+    ./opt/venvs/matrix-synapse/bin/python \
+        -B -m twisted.trial --reporter=text -j2 tests

debian/changelog

@@ -1,3 +1,14 @@
+matrix-synapse-py3 (0.34.0) stable; urgency=medium
+
+  * New synapse release 0.34.0.
+  * Synapse is now installed into a Python 3 virtual environment with
+    up-to-date dependencies.
+  * The matrix-synapse service will now be restarted when the package is
+    upgraded.
+    (Fixes https://github.com/matrix-org/package-synapse-debian/issues/18)
+
+ -- Synapse packaging team <packages@matrix.org>  Wed, 19 Dec 2018 14:00:00 +0000
+
 matrix-synapse (0.33.9-1matrix1) stretch; urgency=medium
 
   [ Erik Johnston ]

debian/control

@@ -1,77 +1,37 @@
-Source: matrix-synapse
-Maintainer: Erik Johnston <erikj@matrix.org>
-Section: python
-Priority: optional
+Source: matrix-synapse-py3
+Section: contrib/python
+Priority: extra
+Maintainer: Synapse Packaging team <packages@matrix.org>
 Build-Depends:
  debhelper (>= 9),
- dh-python,
- dh-systemd (>= 1.5),
- po-debconf,
- python (>= 2.6.6-3),
- python-bcrypt,
- python-blist,
- python-canonicaljson (>=1.1.3),
- python-daemonize,
- python-frozendict (>= 0.4),
- python-lxml,
- python-mock,
- python-msgpack (>=0.3.0),
- python-nacl (>= 0.3.0),
- python-netaddr,
- python-openssl (>= 0.14),
- python-pil,
- python-psutil,
- python-pyasn1,
- python-pydenticon,
- python-pymacaroons-pynacl,
- python-pysaml2,
- python-service-identity (>= 1.0.0),
- python-setuptools (>= 0.6b3),
- python-signedjson (>= 1.0.0),
- python-sortedcontainers,
- python-syutil (>= 0.0.7),
- python-treq (>= 15.1.0),
- python-twisted (>= 17.1.0),
- python-unpaddedbase64 (>= 1.0.1),
- python-yaml,
- python-phonenumbers (>= 8.2.0),
- python-jsonschema (>=2.5.1),
- python-prometheus-client,
- python-attr
-Standards-Version: 3.9.8
-X-Python-Version: >= 2.7
+ dh-systemd,
+ dh-virtualenv (>= 1.0),
+ lsb-release,
+ python3-dev,
+ python3,
+ python3-setuptools,
+ python3-pip,
+ python3-venv,
+ tar,
+Standards-Version: 3.9.5
+Homepage: https://github.com/matrix-org/synapse
 
-Package: matrix-synapse
-Architecture: all
+Package: matrix-synapse-py3
+Architecture: amd64
+Conflicts: matrix-synapse
+Pre-Depends: dpkg (>= 1.16.1)
 Depends:
- ${misc:Depends},
- ${python:Depends},
  adduser,
  debconf,
- lsb-base (>= 3.0-6),
- python-attr (>= 16.0.0),
- python-twisted (>= 17.1.0),
- python-canonicaljson (>=1.1.3),
- python-prometheus-client (>=0.0.14),
+ python3-distutils|libpython3-stdlib (<< 3.6),
+ python3,
+ ${misc:Depends},
+# some of our scripts use perl, but none of them are important,
+# so we put perl:Depends in Suggests rather than Depends.
 Suggests:
- python-bleach (>= 1.4.2),
- python-jinja2 (>= 2.8),
-Recommends:
- python-psycopg2,
- python-lxml,
+ sqlite3,
+ ${perl:Depends},
 Description: Open federated Instant Messaging and VoIP server
  Matrix is an ambitious new ecosystem for open federated Instant
  Messaging and VoIP. Synapse is a reference Matrix server
  implementation.
- .
- Everything in Matrix happens in a room.  Rooms are distributed and do
- not exist on any single server.  Rooms can be located using
- convenience aliases like #matrix:matrix.org or #test:localhost:8448.
- .
- Matrix user IDs look like @matthew:matrix.org (although in the future
- you will normally refer to yourself and others using a 3PID: email
- address, phone number, etc rather than manipulating Matrix user IDs)
- .
- The overall architecture is:
- client <------> homeserver <=============> homeserver <------> client
-                 https://a.org/_matrix      https://b.net/_matrix

debian/copyright

@@ -3,7 +3,7 @@ Upstream-Name: synapse
 Source: https://github.com/matrix-org/synapse
 
 Files: *
-Copyright: 2014-2017, OpenMarket Ltd
+Copyright: 2014-2017, OpenMarket Ltd, 2017-2018 New Vector Ltd
 License: Apache-2.0
 
 Files: synapse/config/saml2.py

debian/gbp.conf

@@ -70,13 +70,9 @@ pid_file: "/var/run/matrix-synapse.pid"
 #
 # cpu_affinity: 0xFFFFFFFF
 
-# Whether to serve a web client from the HTTP/HTTPS root resource.
-web_client: False
-
-# The root directory to server for the above web client.
-# If left undefined, synapse will serve the matrix-angular-sdk web client.
-# Make sure matrix-angular-sdk is installed with pip if web_client is True
-# and web_client_location is undefined
+# The path to the web client which will be served at /_matrix/client/
+# if 'webclient' is configured under the 'listeners' configuration.
+#
 # web_client_location: "/path/to/web/root"
 
 # The public-facing base URL for the client API (not including _matrix/...)

debian/homeserver.yaml

@@ -0,0 +1,4 @@
+opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password
+opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user
+opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db
+opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl

debian/matrix-synapse-py3.links

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+# Attempt to undo some of the braindamage caused by
+# https://github.com/matrix-org/package-synapse-debian/issues/18.
+#
+# Due to reasons [1], the old python2 matrix-synapse package will not stop the
+# service when the package is uninstalled. Our maintainer scripts will do the
+# right thing in terms of ensuring the service is enabled and unmasked, but
+# then do a `systemctl start matrix-synapse`, which of course does nothing -
+# leaving the old (py2) service running.
+#
+# There should normally be no reason for the service to be running during our
+# preinst, so we assume that if it *is* running, it's due to that situation,
+# and stop it.
+#
+# [1] dh_systemd_start doesn't do anything because it sees that there is an
+#     init.d script with the same name, so leaves it to dh_installinit.
+#
+#     dh_installinit doesn't do anything because somebody gave it a --no-start
+#     for unknown reasons.
+
+if [ -x /bin/systemctl ]; then
+    if /bin/systemctl --quiet is-active -- matrix-synapse; then
+        echo >&2 "stopping existing matrix-synapse service"
+        /bin/systemctl stop matrix-synapse || true
+    fi
+fi
+
+#DEBHELPER#
+
+exit 0

debian/postinst debian/matrix-synapse-py3.postinst

@@ -0,0 +1,9 @@
+# Register interest in Python interpreter changes and
+# don't make the Python package dependent on the virtualenv package
+# processing (noawait)
+interest-noawait /usr/bin/python3.5
+interest-noawait /usr/bin/python3.6
+interest-noawait /usr/bin/python3.7
+
+# Also provide a symbolic trigger for all dh-virtualenv packages
+interest dh-virtualenv-interpreter-update

debian/matrix-synapse-py3.preinst

@@ -6,8 +6,8 @@ Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse
-ExecStartPre=/usr/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
-ExecStart=/usr/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
+ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
+ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
 Restart=always
 RestartSec=3
 

debian/matrix-synapse-py3.triggers

@@ -1,18 +1,22 @@
 #!/usr/bin/make -f
+#
+# Build Debian package using https://github.com/spotify/dh-virtualenv
+#
 
-# This file was automatically generated by stdeb 0.8.2 at
-# Fri, 12 Jun 2015 14:32:03 +0100
-export PYBUILD_NAME=matrix-synapse
-%:
-	dh $@ --with python2 --with systemd --buildsystem=pybuild --no-guessing-deps
+override_dh_systemd_enable:
+	dh_systemd_enable --name=matrix-synapse
 
-override_dh_auto_install:
-	python setup.py install --root=debian/matrix-synapse --install-layout=deb
+override_dh_installinit:
+	dh_installinit --name=matrix-synapse
 
-override_dh_auto_build:
+override_dh_strip:
 
-override_dh_installinit:
-	dh_installinit --no-start
+override_dh_shlibdeps:
 
-override_dh_auto_test:
-	PYTHONPATH=. trial tests
+override_dh_virtualenv:
+	./debian/build_virtualenv
+
+# We are restricted to compat level 9 (because xenial), so have to
+# enable the systemd bits manually.
+%:
+	dh $@ --with python-virtualenv --with systemd

debian/matrix-synapse.init

@@ -1 +1 @@
-3.0 (quilt)
+3.0 (native)

debian/matrix-synapse.service

@@ -0,0 +1,35 @@
+# A dockerfile which builds a docker image for building a debian package for
+# synapse. The distro to build for is passed as a docker build var.
+#
+# The default entrypoint expects the synapse source to be mounted as a
+# (read-only) volume at /synapse/source, and an output directory at /debs.
+#
+# A pair of environment variables (TARGET_USERID and TARGET_GROUPID) can be
+# passed to the docker container; if these are set, the build script will chown
+# the build products accordingly, to avoid ending up with things owned by root
+# in the host filesystem.
+
+# Get the distro we want to pull from as a dynamic build variable
+ARG distro=""
+FROM ${distro}
+
+# Install the build dependencies
+RUN apt-get update -qq -o Acquire::Languages=none \
+    && env DEBIAN_FRONTEND=noninteractive apt-get install \
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        debhelper \
+        devscripts \
+        dh-systemd \
+        dh-virtualenv \
+        equivs \
+        lsb-release \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3 \
+        wget
+
+WORKDIR /synapse/source
+ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

debian/patches/0001-tox.patch

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# The script to build the Debian package, as ran inside the Docker image.
+
+set -ex
+
+DIST=`lsb_release -c -s`
+
+# We need to build a newer dh_virtualenv on older OSes like Xenial.
+if [ "$DIST" = 'xenial' ]; then
+    mkdir -p /tmp/dhvenv
+    cd /tmp/dhvenv
+    wget https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz
+    tar xvf 1.1.tar.gz
+    cd dh-virtualenv-1.1/
+    env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io"
+    dpkg-buildpackage -us -uc -b
+    cd /tmp/dhvenv
+    apt-get install -yqq ./dh-virtualenv_1.1-1_all.deb
+fi
+
+
+# we get a read-only copy of the source: make a writeable copy
+cp -aT /synapse/source /synapse/build
+cd /synapse/build
+
+# add an entry to the changelog for this distribution
+dch -M -l "+$DIST" "build for $DIST"
+dch -M -r "" --force-distribution --distribution "$DIST"
+
+dpkg-buildpackage -us -uc
+
+ls -l ..
+
+# copy the build results out, setting perms if necessary
+shopt -s nullglob
+for i in ../*.deb ../*.dsc ../*.tar.xz ../*.changes ../*.buildinfo; do
+    [ -z "$TARGET_USERID" ] || chown "$TARGET_USERID" "$i"
+    [ -z "$TARGET_GROUPID" ] || chgrp "$TARGET_GROUPID" "$i"
+    mv "$i" /debs
+done

debian/patches/0002-change_instructions.patch

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Build the Debian packages using Docker images.
+#
+# This script builds the Docker images and then executes them sequentially, each
+# one building a Debian package for the targeted operating system. It is
+# designed to be a "single command" to produce all the images.
+#
+# By default, builds for all known distributions, but a list of distributions
+# can be passed on the commandline for debugging.
+
+set -ex
+
+cd `dirname $0`
+
+if [ $# -lt 1 ]; then
+    DISTS=(debian:stretch debian:sid ubuntu:xenial ubuntu:bionic ubuntu:cosmic)
+else
+    DISTS=("$@")
+fi
+
+# Make the dir where the debs will live.
+#
+# Note that we deliberately put this outside the source tree, otherwise we tend
+# to get source packages which are full of debs. (We could hack around that
+# with more magic in the build_debian.sh script, but that doesn't solve the
+# problem for natively-run dpkg-buildpakage).
+
+mkdir -p ../../debs
+
+# Build each OS image;
+for i in "${DISTS[@]}"; do
+    TAG=$(echo ${i} | cut -d ":" -f 2)
+    docker build --tag dh-venv-builder:${TAG} --build-arg distro=${i} -f Dockerfile-dhvirtualenv .
+    docker run -it --rm --volume=$(pwd)/../\:/synapse/source:ro --volume=$(pwd)/../../debs:/debs \
+           -e TARGET_USERID=$(id -u) \
+           -e TARGET_GROUPID=$(id -g) \
+           dh-venv-builder:${TAG}
+done

debian/patches/0004-webclient-instructions.patch

@@ -78,7 +78,7 @@
     },
     "postgres": {
         "psycopg2>=2.6": ["psycopg2"]
-    }
+    },
 }
 
 

debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch

@@ -182,7 +182,7 @@ def delete_e2e_room_keys(
 
         keyvalues = {
             "user_id": user_id,
-            "version": version,
+            "version": int(version),
         }
         if room_id:
             keyvalues['room_id'] = room_id

debian/patches/bcrypt.patch

@@ -119,6 +119,7 @@ setenv =
 
 
 [testenv:packaging]
+skip_install=True
 deps =
     check-manifest
 commands =