From 800938116f9ceecb921fec4d700e5ef5577521ca Mon Sep 17 00:00:00 2001 From: Morgan Jones Date: Fri, 8 Nov 2024 21:43:05 -0800 Subject: [PATCH] mattermost: enable test suite, mmctl support, and plugin building Add the following passthrus: - mattermost.withTests (builds with tests but takes a while) - mattermost.withoutTests (builds without tests but is shorter) - mattermost.tests.mattermostWithTests - mattermost.buildPlugin (builds a plugin from source) Enable the mmctl build, and its tests, if selected. --- pkgs/by-name/ma/mattermost/build-plugin.nix | 153 ++++++++ pkgs/by-name/ma/mattermost/package.nix | 170 ++++++--- pkgs/by-name/ma/mattermost/tests.nix | 399 ++++++++++++++++++++ 3 files changed, 670 insertions(+), 52 deletions(-) create mode 100644 pkgs/by-name/ma/mattermost/build-plugin.nix create mode 100644 pkgs/by-name/ma/mattermost/tests.nix diff --git a/pkgs/by-name/ma/mattermost/build-plugin.nix b/pkgs/by-name/ma/mattermost/build-plugin.nix new file mode 100644 index 000000000000..8dd89211e2e2 --- /dev/null +++ b/pkgs/by-name/ma/mattermost/build-plugin.nix @@ -0,0 +1,153 @@ +{ + lib, + stdenv, + mattermost, + nodejs, + writeShellScriptBin, + buildGoModule, + golangci-lint, + gotestsum, + fetchNpmDeps, + npmHooks, + npm-lockfile-fix, +}: + +{ + # The name of the plugin. + pname, + # The plugin version. + version, + # The plugin source. + src, + + # True to ignore golangci-lint warnings. By default set to true + # since plugins warn about the Mattermost interface but build fine. + ignoreGoLintWarnings ? true, + + # The hash of the gomod vendor directory. + vendorHash ? lib.fakeHash, + + # True to build the webapp. + buildWebapp ? true, + + # The NPM dependency hash. + npmDepsHash ? lib.fakeHash, + + # Any extra attributes to pass to buildGoModule. + extraGoModuleAttrs ? { }, +}: + +let + fakeGit = writeShellScriptBin "git" '' + case "$1" in + rev-parse|describe|tag) + echo ${lib.escapeShellArg version} ;; + config) + echo ${lib.escapeShellArg pname} ;; + *) ;; + esac + ''; + + fake-golangci-lint = writeShellScriptBin "golangci-lint" '' + set -uo pipefail + ${lib.getExe golangci-lint} "$@" + result=$? + echo "golangci-lint returned: $result" >&2 + ${lib.optionalString (ignoreGoLintWarnings) '' + if [ $result != 0 ]; then + cat <&2 + Ignoring return value since ignoreGoLintWarnings was true. + Tell this plugin author to fix their lint. + EOF + result=0 + fi + ''} + exit $result + ''; +in +buildGoModule ( + rec { + name = "${pname}-${version}.tar.gz"; + inherit version src vendorHash; + + npmDeps = + if buildWebapp then + fetchNpmDeps { + src = "${src}/webapp"; + hash = npmDepsHash; + forceGitDeps = true; + postFetch = '' + ${lib.getExe npm-lockfile-fix} package-lock.json + ''; + } + else + null; + + makeCacheWritable = true; + forceGitDeps = true; + + overrideModAttrs = final: { + preBuild = '' + go mod tidy + ''; + + # Adding the NPM config hook here breaks things, and isn't even needed. + nativeBuildInputs = lib.lists.remove npmHooks.npmConfigHook final.nativeBuildInputs; + }; + + prePatch = lib.optionalString buildWebapp '' + # Move important node.js files up a level and symlink the originals so the setup hook finds them + for file in package.json package-lock.json node_modules; do + if [ -f "webapp/$file" ]; then + mv "webapp/$file" . + fi + (cd webapp && ln -vsf "../$file") + done + + # Don't allow Go installation in the sandbox, but also don't fail + substituteInPlace Makefile --replace-warn '$(GO) install' '@echo $(GO) install' + ''; + + nativeBuildInputs = [ + fakeGit + nodejs + npmHooks.npmConfigHook + ]; + + buildInputs = [ mattermost ]; + + preBuild = '' + # Or else Babel doesn't run. + export NODE_OPTIONS=--openssl-legacy-provider + + # Only build GOOS and GOARCH plugins so we aren't + # spitting out FreeBSD/Windows/Darwin executables we don't use. + export MM_SERVICESETTINGS_ENABLEDEVELOPER=true + ''; + + buildPhase = '' + runHook preBuild + + # These dependencies are ordinarily fetched via the Makefile, making + # $(GO) install only echo means we still need to install them. + mkdir -p bin + ln -sf ${lib.getExe fake-golangci-lint} bin/golangci-lint + ln -sf ${lib.getExe gotestsum} bin/gotestsum + + # Do the build. + make + + runHook postBuild + ''; + + installPhase = '' + plugin="$(ls dist/*.tar.gz | tail -n1)" + if [ -z "$plugin" ] || [ ! -f "$plugin" ]; then + echo "No plugin tarball in dist folder!" >&2 + exit 1 + fi + cp -av "$plugin" $out + ''; + } + // extraGoModuleAttrs +) diff --git a/pkgs/by-name/ma/mattermost/package.nix b/pkgs/by-name/ma/mattermost/package.nix index 54559408399b..55d2cc7c6488 100644 --- a/pkgs/by-name/ma/mattermost/package.nix +++ b/pkgs/by-name/ma/mattermost/package.nix @@ -1,5 +1,7 @@ { lib, + callPackage, + stdenvNoCC, buildGoModule, fetchFromGitHub, buildNpmPackage, @@ -24,7 +26,60 @@ }, }: -buildGoModule rec { +let + /* + Helper function that sets the `withTests` and `withoutTests` passthru correctly, + and returns the version with tests. + + The primary reason to use this helper over reindenting the whole file is to avoid + lots of manual backporting when the update script runs. + */ + buildMattermost = + { passthru, ... }@args: + let + # Joins the webapp and Matermost derivation together. + # That way patches to the webapp won't cause a rebuild of the server. + wrapMattermost = + server: + stdenvNoCC.mkDerivation { + pname = "${server.pname}-wrapped"; + inherit (server) version; + inherit server; + inherit (server) webapp; + + dontUnpack = true; + + # Just link all the server and webapp root directories together. + installPhase = '' + mkdir -p $out + for dir in "$server" "$webapp"; do + for path in "$dir"/*; do + ln -s "$path" "$out/$(basename -- "$path")" + done + done + ''; + + passthru = finalPassthru // { + inherit server; + inherit (server) webapp; + }; + + inherit (server) meta; + }; + finalPassthru = + let + withoutTestsUnwrapped = buildGoModule (args // { passthru = finalPassthru; }); + withTestsUnwrapped = callPackage ./tests.nix { mattermost = withoutTestsUnwrapped; }; + in + lib.recursiveUpdate passthru rec { + withoutTests = wrapMattermost withoutTestsUnwrapped; + withTests = wrapMattermost withTestsUnwrapped; + tests.mattermostWithTests = withTests; + }; + in + finalPassthru.withTests; +in +buildMattermost rec { pname = "mattermost"; inherit (versionInfo) version; @@ -64,14 +119,12 @@ buildGoModule rec { # We use go 1.22's workspace vendor command, which is not yet available # in the default version of go used in nixpkgs, nor is it used by upstream: # https://github.com/mattermost/mattermost/issues/26221#issuecomment-1945351597 - overrideModAttrs = ( - _: { - buildPhase = '' - make setup-go-work - go work vendor -e - ''; - } - ); + overrideModAttrs = _: { + buildPhase = '' + make setup-go-work + go work vendor -e -v + ''; + }; npmDeps = fetchNpmDeps { inherit src; @@ -81,46 +134,6 @@ buildGoModule rec { forceGitDeps = true; }; - webapp = buildNpmPackage rec { - pname = "mattermost-webapp"; - inherit version src; - - sourceRoot = "${src.name}/webapp"; - - # Remove deprecated image-webpack-loader causing build failures - # See: https://github.com/tcoopman/image-webpack-loader#deprecated - postPatch = '' - substituteInPlace channels/webpack.config.js \ - --replace-fail 'options: {}' 'options: { disable: true }' - ''; - - npmDepsHash = npmDeps.hash; - makeCacheWritable = true; - forceGitDeps = true; - - npmRebuildFlags = [ "--ignore-scripts" ]; - - buildPhase = '' - runHook preBuild - - npm run build --workspace=platform/types - npm run build --workspace=platform/client - npm run build --workspace=platform/components - npm run build --workspace=channels - - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - mkdir -p $out - cp -a channels/dist/* $out - - runHook postInstall - ''; - }; - inherit (versionInfo) vendorHash; modRoot = "./server"; @@ -128,7 +141,10 @@ buildGoModule rec { make setup-go-work ''; - subPackages = [ "cmd/mattermost" ]; + subPackages = [ + "cmd/mattermost" + "cmd/mmctl" + ]; tags = [ "production" ]; @@ -147,8 +163,7 @@ buildGoModule rec { shopt -s extglob mkdir -p $out/{i18n,fonts,templates,config} - # Link in the client and copy the language packs. - ln -sf $webapp $out/client + # Copy the language packs. cp -a $src/server/i18n/* $out/i18n/ # Fonts have the execute bit set, remove it. @@ -162,6 +177,13 @@ buildGoModule rec { go run -tags production ./scripts/config_generator ''; + doInstallCheck = true; + installCheckPhase = '' + for subPackage in $subPackages; do + "$out/bin/$(basename -- "$subPackage")" version | grep "$version" + done + ''; + passthru = { updateScript = nix-update-script { extraArgs = @@ -175,6 +197,50 @@ buildGoModule rec { ]; }; tests.mattermost = nixosTests.mattermost; + + # Builds a Mattermost plugin. + buildPlugin = callPackage ./build-plugin.nix { }; + + # Builds the webapp. + webapp = buildNpmPackage rec { + pname = "mattermost-webapp"; + inherit version src; + + sourceRoot = "${src.name}/webapp"; + + # Remove deprecated image-webpack-loader causing build failures + # See: https://github.com/tcoopman/image-webpack-loader#deprecated + postPatch = '' + substituteInPlace channels/webpack.config.js \ + --replace-fail 'options: {}' 'options: { disable: true }' + ''; + + npmDepsHash = npmDeps.hash; + makeCacheWritable = true; + forceGitDeps = true; + + npmRebuildFlags = [ "--ignore-scripts" ]; + + buildPhase = '' + runHook preBuild + + npm run build --workspace=platform/types + npm run build --workspace=platform/client + npm run build --workspace=platform/components + npm run build --workspace=channels + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/client + cp -a channels/dist/* $out/client + + runHook postInstall + ''; + }; }; meta = with lib; { diff --git a/pkgs/by-name/ma/mattermost/tests.nix b/pkgs/by-name/ma/mattermost/tests.nix new file mode 100644 index 000000000000..3e70be5e5ade --- /dev/null +++ b/pkgs/by-name/ma/mattermost/tests.nix @@ -0,0 +1,399 @@ +{ + lib, + stdenv, + mattermost, + gotestsum, + which, + postgresql, + mariadb, + redis, + curl, + nettools, + runtimeShell, +}: + +let + inherit (lib.lists) optionals; + inherit (lib.strings) versionAtLeast; + is10 = version: versionAtLeast version "10.0"; +in +mattermost.overrideAttrs ( + final: prev: { + doCheck = true; + checkTargets = [ + "test-server" + "test-mmctl" + ]; + nativeCheckInputs = [ + which + postgresql + mariadb + redis + curl + nettools + gotestsum + ]; + + postPatch = + prev.postPatch or "" + + '' + # Just echo install/get/mod commands in the Makefile, since the dependencies are locked. + substituteInPlace server/Makefile \ + --replace-warn '$(GO) install' '@echo $(GO) install' \ + --replace-warn '$(GO) get' '@echo $(GO) get' \ + --replace-warn '$(GO) get' '@echo $(GO) mod' + # mmctl tests shell out by writing a bash script to a tempfile + substituteInPlace server/cmd/mmctl/commands/config_e2e_test.go \ + --replace-fail '#!/bin/bash' '#!${runtimeShell}' + ''; + + # Make sure we disable tests that are broken. + # Use: `nix log | grep FAIL: | awk '{print $3}' | sort` + # and then try to pick the most specific test set to disable, such as: + # X TestFoo + # X TestFoo/TestBar + # -> TestFoo/TestBar/baz_test + disabledTests = + [ + # All these plugin tests for mmctl reach out to the marketplace, which is impossible in the sandbox + "TestMmctlE2ESuite/TestPluginDeleteCmd/Delete_Plugin/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginDeleteCmd/Delete_Plugin/LocalClient" + "TestMmctlE2ESuite/TestPluginDeleteCmd/Delete_a_Plugin_without_permissions" + "TestMmctlE2ESuite/TestPluginDeleteCmd/Delete_Unknown_Plugin/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginDeleteCmd/Delete_Unknown_Plugin/LocalClient" + "TestMmctlE2ESuite/TestPluginInstallURLCmd/install_new_plugins/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginInstallURLCmd/install_new_plugins/LocalClient" + "TestMmctlE2ESuite/TestPluginInstallURLCmd/install_an_already_installed_plugin_without_force/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginInstallURLCmd/install_an_already_installed_plugin_without_force/LocalClient" + "TestMmctlE2ESuite/TestPluginInstallURLCmd/install_an_already_installed_plugin_with_force/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginInstallURLCmd/install_an_already_installed_plugin_with_force/LocalClient" + "TestMmctlE2ESuite/TestPluginMarketplaceInstallCmd/install_a_plugin/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginMarketplaceInstallCmd/install_a_plugin/LocalClient" + "TestMmctlE2ESuite/TestPluginMarketplaceListCmd/List_Marketplace_Plugins_for_Admin_User/SystemAdminClient" + "TestMmctlE2ESuite/TestPluginMarketplaceListCmd/List_Marketplace_Plugins_for_Admin_User/LocalClient" + + # Seems to just be broken. + "TestMmctlE2ESuite/TestPreferenceUpdateCmd" + + # Has a hardcoded "google.com" test which also verifies that the address isn't loopback, + # so we also can't just substituteInPlace to one that will resolve + "TestDialContextFilter" + + # No interfaces but loopback in the sandbox, so returns empty + "TestGetServerIPAddress" + + # S3 bucket tests (needs Minio) + "TestInsecureMakeBucket" + "TestMakeBucket" + "TestListDirectory" + "TestTimeout" + "TestStartServerNoS3Bucket" + "TestS3TestConnection" + "TestS3FileBackendTestSuite" + "TestS3FileBackendTestSuiteWithEncryption" + + # Mail tests (needs a SMTP server) + "TestSendMailUsingConfig" + "TestSendMailUsingConfigAdvanced" + "TestSendMailWithEmbeddedFilesUsingConfig" + "TestSendCloudWelcomeEmail" + "TestMailConnectionAdvanced" + "TestMailConnectionFromConfig" + "TestEmailTest" + "TestBasicAPIPlugins/test_send_mail_plugin" + + # Seems to be unreliable + "TestPluginAPIUpdateUserPreferences" + "TestPluginAPIGetUserPreferences" + + # These invite tests try to send a welcome email and we don't have a SMTP server up. + "TestInviteUsersToTeam" + "TestInviteGuestsToTeam" + "TestSendInviteEmails" + "TestDeliver" + + # https://github.com/mattermost/mattermost/issues/29184 + "TestUpAndDownMigrations/Should_be_reversible_for_mysql" + ] + ++ optionals (is10 final.version) [ + ## mattermostLatest test ignores + + # These bot related tests appear to be broken. + "TestCreateBot" + "TestPatchBot" + "TestGetBot" + "TestEnableBot" + "TestDisableBot" + "TestAssignBot" + "TestConvertBotToUser" + + # Need Elasticsearch or Opensearch + "TestBlevePurgeIndexes" + "TestOpensearchAggregation" + "TestOpensearchInterfaceTestSuite" + "TestOpenSearchIndexerJobIsEnabled" + "TestOpenSearchIndexerPending" + "TestBulkProcessor" + "TestElasticsearchAggregation" + "TestElasticsearchInterfaceTestSuite" + "TestElasticSearchIndexerJobIsEnabled" + "TestElasticSearchIndexerPending" + + # Appear to be broken. + "TestSessionStore/MySQL/SessionGetWithDeviceId" + "TestSessionStore/MySQL/GetMobileSessionMetadata" + ] + ++ optionals (!stdenv.isx86_64) [ + # aarch64: invalid operating system or processor architecture + "TestCanIUpgradeToE0" + ]; + + preCheck = '' + cleanup() { + runHook postCheck + } + trap cleanup EXIT + + # Runs an iteration of the wait loop. + # Returns 0 if we should retry, 1 otherwise. + _wait_loop() { + local process="$1" + local direction="$2" + echo "Waiting for $process to go $direction ($_TRIES attempt(s) left)..." >&2 + _TRIES=$((_TRIES-1)) + if [ $_TRIES -le 0 ]; then + return 1 + else + sleep 1 + return 0 + fi + } + + # Waits on a command named '$1' with direction '$2'. + # The rest of the command is specified in $@. + # If the direction is up, waits for the command to succeed at 1 second intervals. + # If it's down, waits for the command to fail at 1 second intervals. + # Uses a maximum of 5 iterations. Returns 0 if all was ok, or 1 if we timed out. + wait_cmd() { + local process="$1" + local direction="$2" + local tries=10 + _TRIES=$tries + shift; shift + if [ "$direction" == "up" ]; then + while ! "$@" &>/dev/null; do + if ! _wait_loop "$process" "$direction"; then + break + fi + done + else + while "$@" &>/dev/null; do + if ! _wait_loop "$process" "$direction"; then + break + fi + done + fi + + if [ $_TRIES -le 0 ]; then + echo "Timed out after $tries tries" >&2 + return 1 + else + echo "OK, $process went $direction." >&2 + return 0 + fi + } + + # Waits for MySQL to come up or down. + wait_mysql() { + wait_cmd mysql "$1" mysqladmin ping + } + + # Waits for Postgres to come up or down. + wait_postgres() { + wait_cmd postgres "$1" pg_isready -h localhost + } + + # Waits for Redis to come up or down. + wait_redis() { + wait_cmd redis "$1" redis-cli ping + } + + # Starts MySQL. + start_mysql() { + echo "Starting MySQL at $MYSQL_HOME" >&2 + mysqld & + mysql_pid=$! + echo "... PID $mysql_pid" >&2 + wait_mysql up + } + + # Stops MySQL. + stop_mysql() { + if [ "$mysql_pid" -gt 0 ]; then + echo "Terminating MySQL at $MYSQL_HOME (PID $mysql_pid)" >&2 + mysqladmin --host=127.0.0.1 --user=root --password=mostest --wait-for-all-slaves --shutdown-timeout=30 shutdown + wait_mysql down + wait_cmd 'mysql pid' down kill -0 "$mysql_pid" + + # Make sure the worker PID went down too (but it may be already gone). + local worker_pid="$(<"$MYSQL_HOME"/mysqld.pid || echo 0)" + if [ -n "$worker_pid" ] && [ $worker_pid -gt 0 ]; then + wait_cmd 'mysql workers' down kill -0 "$worker_pid" + fi + + mysql_pid=0 + fi + } + + # Starts Postgres. + start_postgres() { + echo "Starting Postgres at $PGDATA" >&2 + pg_ctl start + wait_postgres up + } + + # Stops Postgres. + stop_postgres() { + echo "Terminating Postgres at $PGDATA" >&2 + pg_ctl stop + wait_postgres down + } + + # Starts redis. + start_redis() { + echo "Starting Redis" >&2 + (cd "$REDIS_HOME" && exec redis-server ./redis.conf) & + redis_pid=$! + echo "... PID $redis_pid" >&2 + wait_redis up + } + + # Stops redis. + stop_redis() { + echo "Stopping Redis" >&2 + kill -TERM "$redis_pid" >&2 + wait_redis down + redis_pid=0 + } + + # Configure MySQL. + export MYSQL_HOME="$NIX_BUILD_TOP/.mysql" + mkdir -p "$MYSQL_HOME" + cat <"$MYSQL_HOME/my.cnf" + [client] + port = 3306 + default-character-set = utf8mb4 + socket = $MYSQL_HOME/mysqld.sock + + [mysqld] + skip-host-cache + skip-name-resolve + basedir = ${mariadb} + datadir = $MYSQL_HOME/ + pid-file = $MYSQL_HOME/mysqld.pid + socket = $MYSQL_HOME/mysqld.sock + port = 3306 + explicit_defaults_for_timestamp + collation-server = utf8mb4_general_ci + init-connect = 'SET NAMES utf8mb4' + character-set-server = utf8mb4 + EOF + + # Start MySQL. + mysql_install_db --skip-name-resolve --auth-root-authentication-method=normal + start_mysql + + # Init MySQL. + cat <> "$PGDATA/postgresql.conf" + unix_socket_directories = '$PGDATA/run' + max_connections = 256 + shared_buffers = 96MB + EOF + start_postgres + + # Init Postgres. + cat < "$REDIS_HOME/redis.conf" + bind 127.0.0.1 + port 6379 + protected-mode no + EOF + + # Start Redis. + start_redis + + # Use gotestsum from nixpkgs instead of installing it ourselves. + mkdir -p bin + ln -s "$(which gotestsum)" bin/gotestsum + ''; + + checkPhase = '' + runHook preCheck + + # Ensure we parallelize the tests, and skip the correct ones. + # Spaces are important here due to how the Makefile works. + export GOFLAGS=" -parallel=$NIX_BUILD_CORES -skip='$(echo "$disabledTests" | tr ' ' '|')' " + + # ce n'est pas un conteneur + MMCTL_TESTFLAGS="$GOFLAGS" MM_NO_DOCKER=true make $checkTargets + + runHook postCheck + ''; + + postCheck = '' + # Clean up MySQL. + if [ -d "$MYSQL_HOME" ]; then + stop_mysql + rm -rf "$MYSQL_HOME" + fi + + # Clean up Postgres. + if [ -d "$PGDATA" ]; then + stop_postgres + rm -rf "$PGDATA" + fi + + # Clean up Redis. + if [ -d "$REDIS_HOME" ]; then + stop_redis + rm -rf "$REDIS_HOME" + fi + + # Delete the gotestsum link. + rm -f bin/gotestsum + ''; + } +)