From fd429f5bac9a254f98b86bd8dab301272627ba63 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 15 Mar 2023 13:12:37 +0100 Subject: [PATCH 01/11] matrix-synapse: Prune deps, add missing deps, expose extras Drops extra arguments on the package, as they will be moved over into the module in a more discoverable way next. Also reduces overly broad `with`-scoping. --- pkgs/servers/matrix-synapse/default.nix | 90 +++++++++++++++++++------ 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/pkgs/servers/matrix-synapse/default.nix b/pkgs/servers/matrix-synapse/default.nix index 37629d90e04f..5d774e80ec02 100644 --- a/pkgs/servers/matrix-synapse/default.nix +++ b/pkgs/servers/matrix-synapse/default.nix @@ -1,7 +1,12 @@ -{ lib, stdenv, fetchFromGitHub, python3, openssl, cargo, rustPlatform, rustc -, enableSystemd ? lib.meta.availableOn stdenv.hostPlatform python3.pkgs.systemd +{ lib +, stdenv +, fetchFromGitHub +, python3 +, openssl +, cargo +, rustPlatform +, rustc , nixosTests -, enableRedis ? true , callPackage }: @@ -9,8 +14,7 @@ let plugins = python3.pkgs.callPackage ./plugins { }; tools = callPackage ./tools { }; in -with python3.pkgs; -buildPythonApplication rec { +python3.pkgs.buildPythonApplication rec { pname = "matrix-synapse"; version = "1.87.0"; format = "pyproject"; @@ -34,7 +38,7 @@ buildPythonApplication rec { sed -i '/^setuptools_rust =/d' pyproject.toml ''; - nativeBuildInputs = [ + nativeBuildInputs = with python3.pkgs; [ poetry-core rustPlatform.cargoSetupHook setuptools-rust @@ -42,47 +46,90 @@ buildPythonApplication rec { rustc ]; - buildInputs = [ openssl ]; + buildInputs = [ + openssl + ]; - propagatedBuildInputs = [ - authlib + propagatedBuildInputs = with python3.pkgs; [ + attrs bcrypt bleach canonicaljson - daemonize + cryptography ijson immutabledict jinja2 jsonschema - lxml matrix-common msgpack netaddr + packaging phonenumbers pillow prometheus-client - psutil - psycopg2 pyasn1 + pyasn1-modules pydantic - pyicu pymacaroons - pynacl pyopenssl - pysaml2 pyyaml - requests - setuptools + service-identity signedjson sortedcontainers treq twisted typing-extensions unpaddedbase64 - ] ++ lib.optional enableSystemd systemd - ++ lib.optionals enableRedis [ hiredis txredisapi ]; + ] + ++ twisted.optional-dependencies.tls; - nativeCheckInputs = [ mock parameterized openssl ]; + passthru.optional-dependencies = with python3.pkgs; { + postgres = if isPyPy then [ + psycopg2cffi + ] else [ + psycopg2 + ]; + saml2 = [ + pysaml2 + ]; + oidc = [ + authlib + ]; + systemd = [ + systemd + ]; + url-preview = [ + lxml + ]; + sentry = [ + sentry-sdk + ]; + opentracing = [ + jaeger-client + opentracing + ]; + jwt = [ + authlib + ]; + redis = [ + hiredis + txredisapi + ]; + cache-memory = [ + pympler + ]; + user-search = [ + pyicu + ]; + }; + + nativeCheckInputs = [ + openssl + ] ++ (with python3.pkgs; [ + mock + parameterized + ]) + ++ lib.flatten (lib.attrValues passthru.optional-dependencies); doCheck = !stdenv.isDarwin; @@ -112,6 +159,7 @@ buildPythonApplication rec { meta = with lib; { homepage = "https://matrix.org"; + changelog = "https://github.com/matrix-org/synapse/releases/tag/v${version}"; description = "Matrix reference homeserver"; license = licenses.asl20; maintainers = teams.matrix.members; From 3453128510040c6ad343b98cd44eab5397c63c2e Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Sat, 3 Jun 2023 02:44:30 +0200 Subject: [PATCH 02/11] matrix-synapse: Add wrapper to configure extras The original matrix-synapse packages is now available with the -unwrapped prefix. --- pkgs/servers/matrix-synapse/plugins/ldap3.nix | 4 +- .../plugins/mjolnir-antispam.nix | 4 +- .../plugins/shared-secret-auth.nix | 4 +- pkgs/servers/matrix-synapse/wrapper.nix | 44 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 9 ++-- 5 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 pkgs/servers/matrix-synapse/wrapper.nix diff --git a/pkgs/servers/matrix-synapse/plugins/ldap3.nix b/pkgs/servers/matrix-synapse/plugins/ldap3.nix index b29dc21422e9..feac6f084726 100644 --- a/pkgs/servers/matrix-synapse/plugins/ldap3.nix +++ b/pkgs/servers/matrix-synapse/plugins/ldap3.nix @@ -4,7 +4,7 @@ , fetchPypi , ldap3 , ldaptor -, matrix-synapse +, matrix-synapse-unwrapped , pytestCheckHook , service-identity , setuptools @@ -33,7 +33,7 @@ buildPythonPackage rec { propagatedBuildInputs = [ service-identity ldap3 twisted ]; - nativeCheckInputs = [ ldaptor matrix-synapse pytestCheckHook ]; + nativeCheckInputs = [ ldaptor matrix-synapse-unwrapped pytestCheckHook ]; pythonImportsCheck = [ "ldap_auth_provider" ]; diff --git a/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix b/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix index 2cf0f50901bf..77e3a69d626a 100644 --- a/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix +++ b/pkgs/servers/matrix-synapse/plugins/mjolnir-antispam.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse }: +{ lib, stdenv, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped }: buildPythonPackage rec { pname = "matrix-synapse-mjolnir-antispam"; @@ -13,7 +13,7 @@ buildPythonPackage rec { sourceRoot = "./source/synapse_antispam"; - buildInputs = [ matrix-synapse ]; + buildInputs = [ matrix-synapse-unwrapped ]; doCheck = false; # no tests pythonImportsCheck = [ "mjolnir" ]; diff --git a/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix b/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix index a6e22db34fe5..b5be02a4b21a 100644 --- a/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix +++ b/pkgs/servers/matrix-synapse/plugins/shared-secret-auth.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse, twisted }: +{ lib, buildPythonPackage, fetchFromGitHub, matrix-synapse-unwrapped, twisted }: buildPythonPackage rec { pname = "matrix-synapse-shared-secret-auth"; @@ -14,7 +14,7 @@ buildPythonPackage rec { doCheck = false; pythonImportsCheck = [ "shared_secret_authenticator" ]; - buildInputs = [ matrix-synapse ]; + buildInputs = [ matrix-synapse-unwrapped ]; propagatedBuildInputs = [ twisted ]; meta = with lib; { diff --git a/pkgs/servers/matrix-synapse/wrapper.nix b/pkgs/servers/matrix-synapse/wrapper.nix new file mode 100644 index 000000000000..65f06e808a79 --- /dev/null +++ b/pkgs/servers/matrix-synapse/wrapper.nix @@ -0,0 +1,44 @@ +{ lib +, stdenv +, makeWrapper +, matrix-synapse-unwrapped +, extras ? [ + "postgres" + "url-preview" + "user-search" + ] ++ lib.optional (lib.meta.availableOn stdenv.hostPlatform matrix-synapse-unwrapped.python.pkgs.systemd) "systemd" +, plugins ? [ ] +, ... +}: + +let + extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) extras; + + pluginsEnv = matrix-synapse-unwrapped.python.buildEnv.override { + extraLibs = plugins; + }; + + searchPath = lib.makeSearchPathOutput "lib" matrix-synapse-unwrapped.python.sitePackages (extraPackages ++ [ pluginsEnv ]); +in +stdenv.mkDerivation { + name = (lib.appendToName "wrapped" matrix-synapse-unwrapped).name; + + nativeBuildInputs = [ + makeWrapper + ]; + + buildCommand = '' + for bin in ${matrix-synapse-unwrapped}/bin/*; do + echo $bin + makeWrapper "$bin" "$out/bin/$(basename $bin)" \ + --set PYTHONPATH=${searchPath} + done; + ''; + + passthru = { + unwrapped = matrix-synapse-unwrapped; + + # for backward compatibility + inherit (matrix-synapse-unwrapped) plugins tools; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 39f4d3a3aa04..fc0c92c2ac26 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9624,11 +9624,10 @@ with pkgs; matrix-sliding-sync = callPackage ../servers/matrix-synapse/sliding-sync { }; - matrix-synapse = callPackage ../servers/matrix-synapse { }; - - matrix-synapse-plugins = recurseIntoAttrs matrix-synapse.plugins; - - matrix-synapse-tools = recurseIntoAttrs matrix-synapse.tools; + matrix-synapse = callPackage ../servers/matrix-synapse/wrapper.nix { }; + matrix-synapse-unwrapped = callPackage ../servers/matrix-synapse/default.nix { }; + matrix-synapse-plugins = recurseIntoAttrs matrix-synapse-unwrapped.plugins; + matrix-synapse-tools = recurseIntoAttrs matrix-synapse-unwrapped.tools; matrix-appservice-irc = callPackage ../servers/matrix-synapse/matrix-appservice-irc { }; From 1076c3ada61204581af579474791fc67451a7b39 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 15 Mar 2023 13:22:45 +0100 Subject: [PATCH 03/11] nixos/matrix-synapse: Allow passing extras, discover extras from config With this change we allow the user to configure extras, exposed as optional-dependencies on the matrix-synapse package. The vertical integration between package, user configuration and deployment is a huge boon which then allows us to dynamically adapt the python environment the service runs in, by inspecting the configuration and autodiscovering certain used extras from config paths. --- .../manual/release-notes/rl-2311.section.md | 2 + nixos/modules/services/matrix/synapse.nix | 74 ++++++++++++++++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 5ccaa92914e1..76542e338e34 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -50,6 +50,8 @@ - The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`. +- `matrix-synapse` now refers to a wrapper, encapsulating the original package, which has been renamed to `matrix-synapse-unwrapped`. The arguments `enableSystemd` and `enableRedis` have been removed. All extras can now be configured from [services.matrix-synapse.extras](#opt-services.matrix-synapse.extras), which configures the `extras` argument on the wrapper package. In most cases the required extras will automatically be discovered and installed, when relevant configuration sections are present. + - `etcd` has been updated to 3.5, you will want to read the [3.3 to 3.4](https://etcd.io/docs/v3.5/upgrades/upgrade_3_4/) and [3.4 to 3.5](https://etcd.io/docs/v3.5/upgrades/upgrade_3_5/) upgrade guides - `consul` has been updated to `1.16.0`. See the [release note](https://github.com/hashicorp/consul/releases/tag/v1.16.0) for more details. Once a new Consul version has started and upgraded its data directory, it generally cannot be downgraded to the previous version. diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix index 3dca3ff94f21..5185e2ed8d39 100644 --- a/nixos/modules/services/matrix/synapse.nix +++ b/nixos/modules/services/matrix/synapse.nix @@ -9,11 +9,6 @@ let # remove null values from the final configuration finalSettings = lib.filterAttrsRecursive (_: v: v != null) cfg.settings; configFile = format.generate "homeserver.yaml" finalSettings; - logConfigFile = format.generate "log_config.yaml" cfg.logConfig; - - pluginsEnv = cfg.package.python.buildEnv.override { - extraLibs = cfg.plugins; - }; usePostgresql = cfg.settings.database.name == "psycopg2"; hasLocalPostgresDB = let args = cfg.settings.database.args; in @@ -50,6 +45,30 @@ let "${bindAddress}" }:${builtins.toString listener.port}/" ''; + + defaultExtras = [ + "systemd" + "postgres" + "url-preview" + "user-search" + ]; + + wantedExtras = cfg.extras + ++ lib.optional (cfg.settings ? oidc_providers) "oidc" + ++ lib.optional (cfg.settings ? jwt_config) "jwt" + ++ lib.optional (cfg.settings ? saml2_config) "saml2" + ++ lib.optional (cfg.settings ? opentracing) "opentracing" + ++ lib.optional (cfg.settings ? redis) "redis" + ++ lib.optional (cfg.settings ? sentry) "sentry" + ++ lib.optional (cfg.settings ? user_directory) "user-search" + ++ lib.optional (cfg.settings.url_preview_enabled) "url-preview" + ++ lib.optional (cfg.settings.database.name == "psycopg2") "postgres"; + + wrapped = pkgs.matrix-synapse.override { + matrix-synapse-unwrapped = cfg.package.unwrapped; + extras = wantedExtras; + inherit (cfg) plugins; + }; in { imports = [ @@ -153,8 +172,38 @@ in { type = types.package; default = pkgs.matrix-synapse; defaultText = literalExpression "pkgs.matrix-synapse"; + readOnly = true; description = lib.mdDoc '' - Overridable attribute of the matrix synapse server package to use. + Wrapper package that gets configured through the module. + + If you want to override the unwrapped package use an overlay. + ''; + }; + + extras = mkOption { + type = types.listOf (types.enum (lib.attrNames cfg.package.unwrapped.optional-dependencies)); + default = defaultExtras; + example = literalExpression '' + [ + "cache-memory" # Provide statistics about caching memory consumption + "jwt" # JSON Web Token authentication + "opentracing" # End-to-end tracing support using Jaeger + "oidc" # OpenID Connect authentication + "postgres" # PostgreSQL database backend + "redis" # Redis support for the replication stream between worker processes + "saml2" # SAML2 authentication + "sentry" # Error tracking and performance metrics + "systemd" # Provide the JournalHandler used in the default log_config + "url-preview" # Support for oEmbed URL previews + "user-search" # Support internationalized domain names in user-search + ] + ''; + description = lib.mdDoc '' + Explicitly install extras provided by matrix-synapse. Most + will reconfigure some additional configuration. + + Extras will automatically be enabled, when the relevant + configuration sections are present. ''; }; @@ -193,7 +242,7 @@ in { default = {}; description = mdDoc '' The primary synapse configuration. See the - [sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.version}/docs/sample_config.yaml) + [sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.unwrapped.version}/docs/sample_config.yaml) for possible values. Secrets should be passed in by using the `extraConfigFiles` option. @@ -707,6 +756,9 @@ in { services.matrix-synapse.configFile = configFile; + # default them, so they are additive + services.matrix-synapse.settings.extras = defaultExtras; + users.users.matrix-synapse = { group = "matrix-synapse"; home = cfg.dataDir; @@ -724,14 +776,12 @@ in { after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service"; wantedBy = [ "multi-user.target" ]; preStart = '' - ${cfg.package}/bin/synapse_homeserver \ + ${wrapped}/bin/synapse_homeserver \ --config-path ${configFile} \ --keys-directory ${cfg.dataDir} \ --generate-keys ''; - environment = { - PYTHONPATH = makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ]; - } // optionalAttrs (cfg.withJemalloc) { + environment = optionalAttrs (cfg.withJemalloc) { LD_PRELOAD = "${pkgs.jemalloc}/lib/libjemalloc.so"; }; serviceConfig = { @@ -744,7 +794,7 @@ in { chmod 0600 ${cfg.settings.signing_key_path} '')) ]; ExecStart = '' - ${cfg.package}/bin/synapse_homeserver \ + ${wrapped}/bin/synapse_homeserver \ ${ concatMapStringsSep "\n " (x: "--config-path ${x} \\") ([ configFile ] ++ cfg.extraConfigFiles) } --keys-directory ${cfg.dataDir} ''; From 549bc4bc664e52019a3ce001a205d55a3a3d40ce Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Wed, 14 Jun 2023 14:49:49 +0200 Subject: [PATCH 04/11] nixos/tests/matrix-synapse: Test redis on postgres instance This requires the module to pick up on the redis configuration, and add the required extra packages for redis into the wrapper. --- nixos/tests/matrix/synapse.nix | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nixos/tests/matrix/synapse.nix b/nixos/tests/matrix/synapse.nix index 698d67c793e3..98b077469192 100644 --- a/nixos/tests/matrix/synapse.nix +++ b/nixos/tests/matrix/synapse.nix @@ -65,7 +65,7 @@ in { nodes = { # Since 0.33.0, matrix-synapse doesn't allow underscores in server names - serverpostgres = { pkgs, nodes, ... }: let + serverpostgres = { pkgs, nodes, config, ... }: let mailserverIP = nodes.mailserver.config.networking.primaryIPAddress; in { @@ -77,6 +77,11 @@ in { name = "psycopg2"; args.password = "synapse"; }; + redis = { + enabled = true; + host = "localhost"; + port = config.services.redis.servers.matrix-synapse.port; + }; tls_certificate_path = "${cert}"; tls_private_key_path = "${key}"; registration_shared_secret = registrationSharedSecret; @@ -107,6 +112,11 @@ in { ''; }; + services.redis.servers.matrix-synapse = { + enable = true; + port = 6380; + }; + networking.extraHosts = '' ${mailserverIP} ${mailerDomain} ''; @@ -208,6 +218,9 @@ in { serverpostgres.wait_until_succeeds( "curl --fail -L --cacert ${ca_pem} https://localhost:8448/" ) + serverpostgres.wait_until_succeeds( + "journalctl -u matrix-synapse.service | grep -q 'Connected to redis'" + ) serverpostgres.require_unit_state("postgresql.service") serverpostgres.succeed("register_new_matrix_user -u ${testUser} -p ${testPassword} -a -k ${registrationSharedSecret} https://localhost:8448/") serverpostgres.succeed("obtain-token-and-register-email") From 5a3870c212a0d3f65c26ff599a261b02fe8fac6a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jul 2023 18:01:18 +0200 Subject: [PATCH 05/11] nixos/matrix-synapse: expose final matrix-synapse package via `package`-option When extending this module, it might be necessary to run something from the package that's used in `matrix-synapse.service` (e.g. for workers). Now this can be trivially done by using `config.services.matrix-synapse.package`. Previously it was necessary to reuse the `PYTHONPATH` from the environment of `matrix-synapse.service`, but that one doesn't exist anymore. --- nixos/modules/services/matrix/synapse.nix | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix index 5185e2ed8d39..f516c39d382b 100644 --- a/nixos/modules/services/matrix/synapse.nix +++ b/nixos/modules/services/matrix/synapse.nix @@ -65,7 +65,6 @@ let ++ lib.optional (cfg.settings.database.name == "psycopg2") "postgres"; wrapped = pkgs.matrix-synapse.override { - matrix-synapse-unwrapped = cfg.package.unwrapped; extras = wantedExtras; inherit (cfg) plugins; }; @@ -170,18 +169,27 @@ in { package = mkOption { type = types.package; - default = pkgs.matrix-synapse; - defaultText = literalExpression "pkgs.matrix-synapse"; readOnly = true; description = lib.mdDoc '' - Wrapper package that gets configured through the module. + Reference to the `matrix-synapse` wrapper with all extras + (e.g. for `oidc` or `saml2`) added to the `PYTHONPATH` of all executables. - If you want to override the unwrapped package use an overlay. + This option is useful to reference the "final" `matrix-synapse` package that's + actually used by `matrix-synapse.service`. For instance, when using + workers, it's possible to run + `''${config.services.matrix-synapse.package}/bin/synapse_worker` and + no additional PYTHONPATH needs to be specified for extras or plugins configured + via `services.matrix-synapse`. + + However, this means that this option is supposed to be only declared + by the `services.matrix-synapse` module itself and is thus read-only. + In order to modify `matrix-synapse` itself, use an overlay to override + `pkgs.matrix-synapse-unwrapped`. ''; }; extras = mkOption { - type = types.listOf (types.enum (lib.attrNames cfg.package.unwrapped.optional-dependencies)); + type = types.listOf (types.enum (lib.attrNames pkgs.matrix-synapse-unwrapped.optional-dependencies)); default = defaultExtras; example = literalExpression '' [ @@ -242,7 +250,7 @@ in { default = {}; description = mdDoc '' The primary synapse configuration. See the - [sample configuration](https://github.com/matrix-org/synapse/blob/v${cfg.package.unwrapped.version}/docs/sample_config.yaml) + [sample configuration](https://github.com/matrix-org/synapse/blob/v${pkgs.matrix-synapse-unwrapped.version}/docs/sample_config.yaml) for possible values. Secrets should be passed in by using the `extraConfigFiles` option. @@ -755,6 +763,7 @@ in { ]; services.matrix-synapse.configFile = configFile; + services.matrix-synapse.package = wrapped; # default them, so they are additive services.matrix-synapse.settings.extras = defaultExtras; @@ -776,7 +785,7 @@ in { after = [ "network.target" ] ++ optional hasLocalPostgresDB "postgresql.service"; wantedBy = [ "multi-user.target" ]; preStart = '' - ${wrapped}/bin/synapse_homeserver \ + ${cfg.package}/bin/synapse_homeserver \ --config-path ${configFile} \ --keys-directory ${cfg.dataDir} \ --generate-keys @@ -794,7 +803,7 @@ in { chmod 0600 ${cfg.settings.signing_key_path} '')) ]; ExecStart = '' - ${wrapped}/bin/synapse_homeserver \ + ${cfg.package}/bin/synapse_homeserver \ ${ concatMapStringsSep "\n " (x: "--config-path ${x} \\") ([ configFile ] ++ cfg.extraConfigFiles) } --keys-directory ${cfg.dataDir} ''; From 638460ab9f9ff4db8d3946a946bd5f5f429a2d88 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jul 2023 18:18:35 +0200 Subject: [PATCH 06/11] nixos/release-notes: reword section for synapse wrapper changes --- nixos/doc/manual/release-notes/rl-2311.section.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 76542e338e34..e9da669a93fa 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -50,7 +50,12 @@ - The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`. -- `matrix-synapse` now refers to a wrapper, encapsulating the original package, which has been renamed to `matrix-synapse-unwrapped`. The arguments `enableSystemd` and `enableRedis` have been removed. All extras can now be configured from [services.matrix-synapse.extras](#opt-services.matrix-synapse.extras), which configures the `extras` argument on the wrapper package. In most cases the required extras will automatically be discovered and installed, when relevant configuration sections are present. +- The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though: + - The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below. + - The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed. + - These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["postgres"]; }` for `psycopg2` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if [`services.matrix-synapse.settings.database.name`](#opt-services.matrix-synapse.settings.database.name) is `psycopg2`, `"postgres"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`. + - A list of all extras (and the extras enabled by default) can be found at the [option's reference for `services.matrix-synapse.extras`](#opt-services.matrix-synapse.extras). + - In some cases (e.g. for running synapse workers) it was necessary to re-use the `PYTHONPATH` of `matrix-synapse.service`'s environment to have all plugins available. This isn't necessary anymore, instead `config.services.matrix-synapse.package` can be used as it points to the wrapper with properly configured `extras` and also all plugins defined via [`services.matrix-synapse.plugins`](#opt-services.matrix-synapse.plugins) available. This is also the reason for why the option is read-only now, it's supposed to be set by the module only. - `etcd` has been updated to 3.5, you will want to read the [3.3 to 3.4](https://etcd.io/docs/v3.5/upgrades/upgrade_3_4/) and [3.4 to 3.5](https://etcd.io/docs/v3.5/upgrades/upgrade_3_5/) upgrade guides From 5bf466dae98770bc36fde98eb515bb2e58e0e1cb Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jul 2023 18:46:43 +0200 Subject: [PATCH 07/11] matrix-synapse: fix PYTHONPATH wrapper --- pkgs/servers/matrix-synapse/wrapper.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/servers/matrix-synapse/wrapper.nix b/pkgs/servers/matrix-synapse/wrapper.nix index 65f06e808a79..6be1e7af7fab 100644 --- a/pkgs/servers/matrix-synapse/wrapper.nix +++ b/pkgs/servers/matrix-synapse/wrapper.nix @@ -31,7 +31,7 @@ stdenv.mkDerivation { for bin in ${matrix-synapse-unwrapped}/bin/*; do echo $bin makeWrapper "$bin" "$out/bin/$(basename $bin)" \ - --set PYTHONPATH=${searchPath} + --set PYTHONPATH ${searchPath} done; ''; From 190886c5cca51bee671c5f3329ce03e119768029 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jul 2023 18:54:17 +0200 Subject: [PATCH 08/11] nixos/matrix-synapse: clarify that `extras` are additive --- nixos/modules/services/matrix/synapse.nix | 4 ++++ pkgs/servers/matrix-synapse/wrapper.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix index f516c39d382b..4bcad572d069 100644 --- a/nixos/modules/services/matrix/synapse.nix +++ b/nixos/modules/services/matrix/synapse.nix @@ -212,6 +212,10 @@ in { Extras will automatically be enabled, when the relevant configuration sections are present. + + Please note that this option is additive: i.e. when adding a new item + to this list, the defaults are still kept. To override the defaults as well, + use `lib.mkForce`. ''; }; diff --git a/pkgs/servers/matrix-synapse/wrapper.nix b/pkgs/servers/matrix-synapse/wrapper.nix index 6be1e7af7fab..930142c5f55e 100644 --- a/pkgs/servers/matrix-synapse/wrapper.nix +++ b/pkgs/servers/matrix-synapse/wrapper.nix @@ -12,7 +12,7 @@ }: let - extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) extras; + extraPackages = lib.concatMap (extra: matrix-synapse-unwrapped.optional-dependencies.${extra}) (lib.unique extras); pluginsEnv = matrix-synapse-unwrapped.python.buildEnv.override { extraLibs = plugins; From 701d0e1da6372db7951025ff0b24ec48e82453d8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jul 2023 19:15:09 +0200 Subject: [PATCH 09/11] nixos/matrix-synapse: fix path to extras for additive settings --- nixos/modules/services/matrix/synapse.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix index 4bcad572d069..75c28d13a018 100644 --- a/nixos/modules/services/matrix/synapse.nix +++ b/nixos/modules/services/matrix/synapse.nix @@ -770,7 +770,7 @@ in { services.matrix-synapse.package = wrapped; # default them, so they are additive - services.matrix-synapse.settings.extras = defaultExtras; + services.matrix-synapse.extras = defaultExtras; users.users.matrix-synapse = { group = "matrix-synapse"; From 9f6ed8c2b26ad960b40c652dcc3a744e954ee449 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jul 2023 19:27:22 +0200 Subject: [PATCH 10/11] nixos/release-notes: use redis as example for extras in synapse postgres isn't such a good idea actually because it's added by default to the wrapper. --- nixos/doc/manual/release-notes/rl-2311.section.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index e9da669a93fa..ece6b98dfef1 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -53,7 +53,7 @@ - The `matrix-synapse` package & module have undergone some significant internal changes, for most setups no intervention is needed, though: - The option [`services.matrix-synapse.package`](#opt-services.matrix-synapse.package) is now read-only. For modifying the package, use an overlay which modifies `matrix-synapse-unwrapped` instead. More on that below. - The `enableSystemd` & `enableRedis` arguments have been removed and `matrix-synapse` has been renamed to `matrix-synapse-unwrapped`. Also, several optional dependencies (such as `psycopg2` or `authlib`) have been removed. - - These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["postgres"]; }` for `psycopg2` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if [`services.matrix-synapse.settings.database.name`](#opt-services.matrix-synapse.settings.database.name) is `psycopg2`, `"postgres"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`. + - These optional dependencies are automatically added via a wrapper (`pkgs.matrix-synapse.override { extras = ["redis"]; }` for `hiredis` & `txredisapi` for instance) if the relevant config section is declared in `services.matrix-synapse.settings`. For instance, if `services.matrix-synapse.settings.redis.enabled` is set to `true`, `"redis"` will be automatically added to the `extras` list of `pkgs.matrix-synapse`. - A list of all extras (and the extras enabled by default) can be found at the [option's reference for `services.matrix-synapse.extras`](#opt-services.matrix-synapse.extras). - In some cases (e.g. for running synapse workers) it was necessary to re-use the `PYTHONPATH` of `matrix-synapse.service`'s environment to have all plugins available. This isn't necessary anymore, instead `config.services.matrix-synapse.package` can be used as it points to the wrapper with properly configured `extras` and also all plugins defined via [`services.matrix-synapse.plugins`](#opt-services.matrix-synapse.plugins) available. This is also the reason for why the option is read-only now, it's supposed to be set by the module only. From d2facca5c0c776bef2e142e129df90355ec80a27 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 1 Aug 2023 18:16:56 +0200 Subject: [PATCH 11/11] nixos/matrix-synapse: fix option description of `extras` option Co-authored-by: Benjamin Saunders --- nixos/modules/services/matrix/synapse.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/matrix/synapse.nix b/nixos/modules/services/matrix/synapse.nix index 75c28d13a018..ef69a8adebb0 100644 --- a/nixos/modules/services/matrix/synapse.nix +++ b/nixos/modules/services/matrix/synapse.nix @@ -208,7 +208,7 @@ in { ''; description = lib.mdDoc '' Explicitly install extras provided by matrix-synapse. Most - will reconfigure some additional configuration. + will require some additional configuration. Extras will automatically be enabled, when the relevant configuration sections are present.