diff --git a/nixos/modules/services/web-apps/mattermost.nix b/nixos/modules/services/web-apps/mattermost.nix index b1352778fee0..22dad6497d07 100644 --- a/nixos/modules/services/web-apps/mattermost.nix +++ b/nixos/modules/services/web-apps/mattermost.nix @@ -147,34 +147,34 @@ let else throw "Invalid database driver: ${cfg.database.driver}"; - mattermostPluginDerivations = - with pkgs; - map ( - plugin: - stdenv.mkDerivation { - name = "mattermost-plugin"; - installPhase = '' - mkdir -p $out/share - cp ${plugin} $out/share/plugin.tar.gz - ''; - dontUnpack = true; - dontPatch = true; - dontConfigure = true; - dontBuild = true; - preferLocalBuild = true; - } - ) cfg.plugins; + mattermostPluginDerivations = map ( + plugin: + pkgs.stdenvNoCC.mkDerivation { + name = "${cfg.package.name}-plugin"; + installPhase = '' + runHook preInstall + mkdir -p $out/share + ln -sf ${plugin} $out/share/plugin.tar.gz + runHook postInstall + ''; + dontUnpack = true; + dontPatch = true; + dontConfigure = true; + dontBuild = true; + preferLocalBuild = true; + } + ) cfg.plugins; mattermostPlugins = - with pkgs; if mattermostPluginDerivations == [ ] then null else - stdenv.mkDerivation { + pkgs.stdenvNoCC.mkDerivation { name = "${cfg.package.name}-plugins"; - nativeBuildInputs = [ autoPatchelfHook ] ++ mattermostPluginDerivations; + nativeBuildInputs = [ pkgs.autoPatchelfHook ] ++ mattermostPluginDerivations; buildInputs = [ cfg.package ]; installPhase = '' + runHook preInstall mkdir -p $out plugins=(${ escapeShellArgs (map (plugin: "${plugin}/share/plugin.tar.gz") mattermostPluginDerivations) @@ -187,6 +187,7 @@ let GZIP_OPT=-9 tar -C "$hash" -cvzf "$out/$hash.tar.gz" . rm -rf "$hash" done + runHook postInstall ''; dontUnpack = true; @@ -254,8 +255,8 @@ let } ); - mattermostConfJSON = pkgs.writeText "mattermost-config.json" (builtins.toJSON mattermostConf); - + format = pkgs.formats.json { }; + finalConfig = format.generate "mattermost-config.json" mattermostConf; in { imports = [ @@ -454,9 +455,9 @@ in the options specified in services.mattermost will be generated but won't be overwritten on changes or rebuilds. - If this option is disabled, changes in the system console won't - be possible (default). If an config.json is present, it will be - overwritten! + If this option is disabled, persistent changes in the system + console won't be possible (the default). If a config.json is + present, it will be overwritten at service start! ''; }; @@ -480,7 +481,20 @@ in description = '' Plugins to add to the configuration. Overrides any installed if non-null. This is a list of paths to .tar.gz files or derivations evaluating to - .tar.gz files. + .tar.gz files. You can use `mattermost.buildPlugin` to build plugins; + see the NixOS documentation for more details. + ''; + }; + + pluginsBundle = mkOption { + type = with types; nullOr package; + default = mattermostPlugins; + defaultText = '' + All entries in {config}`services.mattermost.plugins`, repacked + ''; + description = '' + Derivation building to a directory of plugin tarballs. + This overrides {option}`services.mattermost.plugins` if provided. ''; }; @@ -508,7 +522,8 @@ in type = with types; attrsOf (either int str); default = { }; description = '' - Extra environment variables to export to the Mattermost process, in the systemd unit. + Extra environment variables to export to the Mattermost process + from the systemd unit configuration. ''; example = { MM_SERVICESETTINGS_SITEURL = "http://example.com"; @@ -524,11 +539,11 @@ in for mattermost (see [the Mattermost documentation](https://docs.mattermost.com/configure/configuration-settings.html#environment-variables)). Settings defined in the environment file will overwrite settings - set via nix or via the {option}`services.mattermost.extraConfig` + set via Nix or via the {option}`services.mattermost.extraConfig` option. Useful for setting config options without their value ending up in the - (world-readable) nix store, e.g. for a database password. + (world-readable) Nix store, e.g. for a database password. ''; }; @@ -687,7 +702,7 @@ in }; settings = mkOption { - type = types.attrs; + inherit (format) type; default = { }; description = '' Additional configuration options as Nix attribute set in config.json schema. @@ -786,7 +801,7 @@ in "d= ${tempDir} 0750 ${cfg.user} ${cfg.group} - -" # Ensure that pluginDir is a directory, as it could be a symlink on prior versions. - "r- ${pluginDir} - - - - -" + # Don't remove or clean it out since it should be persistent, as this is where plugins are unpacked. "d= ${pluginDir} 0750 ${cfg.user} ${cfg.group} - -" # Ensure that the plugin directories exist. @@ -801,15 +816,14 @@ in "L+ ${cfg.dataDir}/client - - - - ${cfg.package}/client" ] ++ ( - if mattermostPlugins == null then - # Create the plugin tarball directory if it's a symlink. + if cfg.pluginsBundle == null then + # Create the plugin tarball directory to allow plugin uploads. [ - "r- ${cfg.dataDir}/plugins - - - - -" "d= ${cfg.dataDir}/plugins 0750 ${cfg.user} ${cfg.group} - -" ] else - # Symlink the plugin tarball directory, removing anything existing. - [ "L+ ${cfg.dataDir}/plugins - - - - ${mattermostPlugins}" ] + # Symlink the plugin tarball directory, removing anything existing, since it's managed by Nix. + [ "L+ ${cfg.dataDir}/plugins - - - - ${cfg.pluginsBundle}" ] ); systemd.services.mattermost = rec { @@ -836,7 +850,7 @@ in configDir=${escapeShellArg cfg.configDir} logDir=${escapeShellArg cfg.logDir} package=${escapeShellArg cfg.package} - nixConfig=${escapeShellArg mattermostConfJSON} + nixConfig=${escapeShellArg finalConfig} '' + optionalString (versionAtLeast config.system.stateVersion "25.05") '' # Migrate configs in the pre-25.05 directory structure. diff --git a/nixos/tests/mattermost/default.nix b/nixos/tests/mattermost/default.nix index eb7b814e3983..47efc213383d 100644 --- a/nixos/tests/mattermost/default.nix +++ b/nixos/tests/mattermost/default.nix @@ -33,7 +33,7 @@ import ../make-test-python.nix ( ); }; - system.stateVersion = lib.mkDefault "25.05"; + system.stateVersion = lib.mkDefault (lib.versions.majorMinor lib.version); services.mattermost = lib.recursiveUpdate { enable = true; @@ -63,7 +63,7 @@ import ../make-test-python.nix ( # Upgrade to the latest Mattermost. specialisation.latest.configuration = { services.mattermost.package = lib.mkForce pkgs.mattermostLatest; - system.stateVersion = lib.mkVMOverride "25.05"; + system.stateVersion = lib.mkVMOverride (lib.versions.majorMinor lib.version); }; } ) @@ -90,57 +90,60 @@ import ../make-test-python.nix ( name = "mattermost"; nodes = rec { - postgresMutable = + postgresMutable = makeMattermost { + mutableConfig = true; + preferNixConfig = false; + settings.SupportSettings.HelpLink = "https://search.nixos.org"; + } { }; + postgresMostlyMutable = makeMattermost { mutableConfig = true; - preferNixConfig = false; - settings.SupportSettings.HelpLink = "https://search.nixos.org"; + preferNixConfig = true; + plugins = with pkgs; [ + # Build the demo plugin. + (mattermost.buildPlugin { + pname = "mattermost-plugin-starter-template"; + version = "0.1.0"; + src = fetchFromGitHub { + owner = "mattermost"; + repo = "mattermost-plugin-starter-template"; + # Newer versions have issues with their dependency lockfile. + rev = "7c98e89ac1a268ce8614bc665571b7bbc9a70df2"; + hash = "sha256-uyfxB0GZ45qL9ssWUord0eKQC6S0TlCTtjTOXWtK4H0="; + }; + vendorHash = "sha256-Jl4F9YkHNqiFP9/yeyi4vTntqxMk/J1zhEP6QLSvJQA="; + npmDepsHash = "sha256-z08nc4XwT+uQjQlZiUydJyh8mqeJoYdPFWuZpw9k99s="; + }) + + # Build the todos plugin. + (mattermost.buildPlugin { + pname = "mattermost-plugin-todo"; + version = "0.8-pre"; + src = fetchFromGitHub { + owner = "mattermost-community"; + repo = "mattermost-plugin-todo"; + # 0.7.1 didn't work, seems to use an older set of node dependencies. + rev = "f25dc91ea401c9f0dcd4abcebaff10eb8b9836e5"; + hash = "sha256-OM+m4rTqVtolvL5tUE8RKfclqzoe0Y38jLU60Pz7+HI="; + }; + vendorHash = "sha256-5KpechSp3z/Nq713PXYruyNxveo6CwrCSKf2JaErbgg="; + npmDepsHash = "sha256-o2UOEkwb8Vx2lDWayNYgng0GXvmS6lp/ExfOq3peyMY="; + extraGoModuleAttrs = { + npmFlags = [ "--legacy-peer-deps" ]; + }; + }) + ]; } { # Last version to support the "old" config layout. system.stateVersion = lib.mkForce "24.11"; - # First version to support the "new" config layout. - specialisation.upgrade.configuration.system.stateVersion = lib.mkVMOverride "25.05"; + # Supports the "new" config layout. + specialisation.upgrade.configuration.system.stateVersion = lib.mkVMOverride ( + lib.versions.majorMinor lib.version + ); }; - postgresMostlyMutable = makeMattermost { - mutableConfig = true; - plugins = with pkgs; [ - # Build the demo plugin. - (mattermost.buildPlugin { - pname = "mattermost-plugin-starter-template"; - version = "0.1.0"; - src = fetchFromGitHub { - owner = "mattermost"; - repo = "mattermost-plugin-starter-template"; - # Newer versions have issues with their dependency lockfile. - rev = "7c98e89ac1a268ce8614bc665571b7bbc9a70df2"; - hash = "sha256-uyfxB0GZ45qL9ssWUord0eKQC6S0TlCTtjTOXWtK4H0="; - }; - vendorHash = "sha256-Jl4F9YkHNqiFP9/yeyi4vTntqxMk/J1zhEP6QLSvJQA="; - npmDepsHash = "sha256-z08nc4XwT+uQjQlZiUydJyh8mqeJoYdPFWuZpw9k99s="; - }) - - # Build the todos plugin. - (mattermost.buildPlugin { - pname = "mattermost-plugin-todo"; - version = "0.8-pre"; - src = fetchFromGitHub { - owner = "mattermost-community"; - repo = "mattermost-plugin-todo"; - # 0.7.1 didn't work, seems to use an older set of node dependencies. - rev = "f25dc91ea401c9f0dcd4abcebaff10eb8b9836e5"; - hash = "sha256-OM+m4rTqVtolvL5tUE8RKfclqzoe0Y38jLU60Pz7+HI="; - }; - vendorHash = "sha256-5KpechSp3z/Nq713PXYruyNxveo6CwrCSKf2JaErbgg="; - npmDepsHash = "sha256-o2UOEkwb8Vx2lDWayNYgng0GXvmS6lp/ExfOq3peyMY="; - extraGoModuleAttrs = { - npmFlags = [ "--legacy-peer-deps" ]; - }; - }) - ]; - } { }; postgresImmutable = makeMattermost { package = pkgs.mattermost.overrideAttrs (prev: { webapp = prev.webapp.overrideAttrs (prevWebapp: { @@ -343,9 +346,14 @@ import ../make-test-python.nix ( ''; in '' + import sys import shlex + import threading + import queue def wait_mattermost_up(node, site_name="${siteName}"): + print(f"wait_mattermost_up({node.name!r}, site_name={site_name!r})", file=sys.stderr) + node.wait_for_unit("multi-user.target") node.systemctl("start mattermost.service") node.wait_for_unit("mattermost.service") node.wait_for_open_port(8065) @@ -353,20 +361,25 @@ import ../make-test-python.nix ( node.succeed(f"curl {shlex.quote('${url}')}/index.html | grep {shlex.quote(site_name)}") def restart_mattermost(node, site_name="${siteName}"): + print(f"restart_mattermost({node.name!r}, site_name={site_name!r})", file=sys.stderr) node.systemctl("restart mattermost.service") wait_mattermost_up(node, site_name) def expect_config(node, mattermost_version, *configs): + print(f"expect_config({node.name!r}, {mattermost_version!r}, *{configs!r})", file=sys.stderr) for config in configs: node.succeed(f"${expectConfig} {shlex.quote(config)} {shlex.quote(mattermost_version)}") def expect_plugins(node, jq_or_code): + print(f"expect_plugins({node.name!r}, {jq_or_code!r})", file=sys.stderr) node.succeed(f"${expectPlugins} {shlex.quote(str(jq_or_code))}") def ensure_post(node, fail_if_not_found=False): + print(f"ensure_post({node.name!r}, fail_if_not_found={fail_if_not_found!r})", file=sys.stderr) node.succeed(f"${ensurePost} {shlex.quote('${url}')} {1 if fail_if_not_found else 0}") - def set_config(node, *configs, nixos_version='25.05'): + def set_config(node, *configs, nixos_version='${lib.versions.majorMinor lib.version}'): + print(f"set_config({node.name!r}, *{configs!r}, nixos_version={nixos_version!r})", file=sys.stderr) for config in configs: args = [shlex.quote("${setConfig}")] args.append(shlex.quote(config)) @@ -374,8 +387,13 @@ import ../make-test-python.nix ( args.append(shlex.quote(str(nixos_version))) node.succeed(' '.join(args)) - def run_mattermost_tests(mutableToplevel: str, mutable, - mostlyMutableToplevel: str, mostlyMutable, + def switch_to_specialisation(node, toplevel: str, specialisation: str): + print(f"switch_to_specialisation({node.name!r}, {toplevel!r}, {specialisation!r})", file=sys.stderr) + node.succeed(f"{toplevel}/specialisation/{specialisation}/bin/switch-to-configuration switch || true") + + def run_mattermost_tests(shutdown_queue: queue.Queue, + mutableToplevel: str, mutable, + mostlyMutableToplevel: str, mostlyMutablePlugins: str, mostlyMutable, immutableToplevel: str, immutable, environmentFileToplevel: str, environmentFile): esr, latest = '${pkgs.mattermost.version}', '${pkgs.mattermostLatest.version}' @@ -391,8 +409,7 @@ import ../make-test-python.nix ( set_config( mutable, '.SupportSettings.AboutLink = "https://mattermost.com"', - '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"', - nixos_version='24.11' # Default 'mutable' config is an old version + '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"' ) ensure_post(mutable) restart_mattermost(mutable) @@ -401,23 +418,14 @@ import ../make-test-python.nix ( expect_config(mutable, esr, '.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"') ensure_post(mutable, fail_if_not_found=True) - # Switch to the newer config - mutable.succeed(f"{mutableToplevel}/specialisation/upgrade/bin/switch-to-configuration switch") - wait_mattermost_up(mutable) - - # AboutLink and HelpLink should be changed, still, and the post should still exist - expect_config(mutable, esr, '.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"') - ensure_post(mutable, fail_if_not_found=True) - # Switch to the latest Mattermost version - mutable.succeed(f"{mutableToplevel}/specialisation/latest/bin/switch-to-configuration switch") + switch_to_specialisation(mutable, mutableToplevel, "latest") wait_mattermost_up(mutable) # AboutLink and HelpLink should be changed, still, and the post should still exist expect_config(mutable, latest, '.AboutLink == "https://mattermost.com" and .HelpLink == "https://nixos.org/nixos/manual"') ensure_post(mutable, fail_if_not_found=True) - - mutable.shutdown() + shutdown_queue.put(mutable) ## Mostly mutable node tests ## mostlyMutable.start() @@ -434,13 +442,40 @@ import ../make-test-python.nix ( mostlyMutable, '.SupportSettings.AboutLink = "https://mattermost.com"', '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual"', + nixos_version='24.11' # Default 'mostlyMutable' config is an old version + ) + ensure_post(mostlyMutable) + restart_mattermost(mostlyMutable) + + # HelpLink should be changed but AboutLink should not, and the post should exist + expect_config(mostlyMutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"') + ensure_post(mostlyMutable, fail_if_not_found=True) + + # Switch to the newer config and make sure the plugins directory is replaced with a directory, + # since it could have been a symlink on previous versions. + mostlyMutable.systemctl("stop mattermost.service") + mostlyMutable.succeed(f"[ ! -L /var/lib/mattermost/data/plugins ] && rm -rf /var/lib/mattermost/data/plugins && ln -s {mostlyMutablePlugins} /var/lib/mattermost/data/plugins || true") + mostlyMutable.succeed('[ -L /var/lib/mattermost/data/plugins ] && [ -d /var/lib/mattermost/data/plugins ]') + switch_to_specialisation(mostlyMutable, mostlyMutableToplevel, "upgrade") + wait_mattermost_up(mostlyMutable) + mostlyMutable.succeed('[ ! -L /var/lib/mattermost/data/plugins ] && [ -d /var/lib/mattermost/data/plugins ]') + + # HelpLink should be changed, still, and the post should still exist + expect_config(mostlyMutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"') + ensure_post(mostlyMutable, fail_if_not_found=True) + + # Edit the config and make a post + set_config( + mostlyMutable, + '.SupportSettings.AboutLink = "https://mattermost.com/foo"', + '.SupportSettings.HelpLink = "https://nixos.org/nixos/manual/bar"', '.PluginSettings.PluginStates."com.mattermost.plugin-todo".Enable = true' ) ensure_post(mostlyMutable) restart_mattermost(mostlyMutable) # AboutLink should be overridden by NixOS configuration; HelpLink should be what we set above - expect_config(mostlyMutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"') + expect_config(mostlyMutable, esr, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual/bar"') # Single plugin that's now enabled. expect_plugins(mostlyMutable, 'length == 1') @@ -449,14 +484,14 @@ import ../make-test-python.nix ( ensure_post(mostlyMutable, fail_if_not_found=True) # Switch to the latest Mattermost version - mostlyMutable.succeed(f"{mostlyMutableToplevel}/specialisation/latest/bin/switch-to-configuration switch") + switch_to_specialisation(mostlyMutable, mostlyMutableToplevel, "latest") wait_mattermost_up(mostlyMutable) # AboutLink should be overridden and the post should still exist - expect_config(mostlyMutable, latest, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual"') + expect_config(mostlyMutable, latest, '.AboutLink == "https://nixos.org" and .HelpLink == "https://nixos.org/nixos/manual/bar"') ensure_post(mostlyMutable, fail_if_not_found=True) - mostlyMutable.shutdown() + shutdown_queue.put(mostlyMutable) ## Immutable node tests ## immutable.start() @@ -484,14 +519,14 @@ import ../make-test-python.nix ( ensure_post(immutable, fail_if_not_found=True) # Switch to the latest Mattermost version - immutable.succeed(f"{immutableToplevel}/specialisation/latest/bin/switch-to-configuration switch") + switch_to_specialisation(immutable, immutableToplevel, "latest") wait_mattermost_up(immutable) # AboutLink and HelpLink should be changed, still, and the post should still exist expect_config(immutable, latest, '.AboutLink == "https://nixos.org" and .HelpLink == "https://search.nixos.org"') ensure_post(immutable, fail_if_not_found=True) - immutable.shutdown() + shutdown_queue.put(immutable) ## Environment File node tests ## environmentFile.start() @@ -503,19 +538,31 @@ import ../make-test-python.nix ( ensure_post(environmentFile, fail_if_not_found=True) # Switch to the latest Mattermost version - environmentFile.succeed(f"{environmentFileToplevel}/specialisation/latest/bin/switch-to-configuration switch") + switch_to_specialisation(environmentFile, environmentFileToplevel, "latest") wait_mattermost_up(environmentFile) # AboutLink should be changed still, and the post should still exist expect_config(environmentFile, latest, '.AboutLink == "https://nixos.org"') ensure_post(environmentFile, fail_if_not_found=True) - environmentFile.shutdown() + shutdown_queue.put(environmentFile) + + # Run shutdowns asynchronously so we can pipeline them. + shutdown_queue: queue.Queue = queue.Queue() + def shutdown_worker(): + while True: + node = shutdown_queue.get() + print(f"Shutting down node {node.name!r} asynchronously", file=sys.stderr) + node.shutdown() + shutdown_queue.task_done() + threading.Thread(target=shutdown_worker, daemon=True).start() run_mattermost_tests( + shutdown_queue, "${nodes.mysqlMutable.system.build.toplevel}", mysqlMutable, "${nodes.mysqlMostlyMutable.system.build.toplevel}", + "${nodes.mysqlMostlyMutable.services.mattermost.pluginsBundle}", mysqlMostlyMutable, "${nodes.mysqlImmutable.system.build.toplevel}", mysqlImmutable, @@ -524,15 +571,20 @@ import ../make-test-python.nix ( ) run_mattermost_tests( + shutdown_queue, "${nodes.postgresMutable.system.build.toplevel}", postgresMutable, "${nodes.postgresMostlyMutable.system.build.toplevel}", + "${nodes.postgresMostlyMutable.services.mattermost.pluginsBundle}", postgresMostlyMutable, "${nodes.postgresImmutable.system.build.toplevel}", postgresImmutable, "${nodes.postgresEnvironmentFile.system.build.toplevel}", postgresEnvironmentFile ) + + # Drain the queue + shutdown_queue.join() ''; } )