From 75f559665227c3b20c36babd44b33619c8318cd2 Mon Sep 17 00:00:00 2001 From: sodiboo Date: Sat, 12 Jul 2025 03:09:36 +0200 Subject: [PATCH 1/2] nixos/tests/meilisearch: fix race condition when creating index previously, when creating the index, there is a race condition between the task being created and when we check if the index was successfully created. if it is created too slow, the test fails. it already polled/awaited when adding documents, so i generalized this logic (and made it terminate anytime the task finishes, even if that wasn't success). not ideal, since meilisearch supports proper webhook callbacks, but whatever, we're writing tests, not production code. --- nixos/tests/meilisearch.nix | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/nixos/tests/meilisearch.nix b/nixos/tests/meilisearch.nix index ae88b8e07157..9a27fd05800c 100644 --- a/nixos/tests/meilisearch.nix +++ b/nixos/tests/meilisearch.nix @@ -35,27 +35,26 @@ in machine.wait_for_unit("meilisearch") machine.wait_for_open_port(7700) + def wait_task(cmd): + response = json.loads(machine.succeed(cmd)) + task_uid = response["taskUid"] + machine.wait_until_succeeds( + f"curl ${apiUrl}/tasks/{task_uid} | jq -e '.status | IN(\"succeeded\", \"failed\", \"canceled\")'" + ) + machine.succeed(f"curl ${apiUrl}/tasks/{task_uid} | jq -e '.status == \"succeeded\"'") + return response + with subtest("check version"): version = json.loads(machine.succeed("curl ${apiUrl}/version")) assert version["pkgVersion"] == "${pkgs.meilisearch.version}" with subtest("create index"): - machine.succeed( - "curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes --data @${indexJSON}" - ) + wait_task("curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes --data @${indexJSON}") indexes = json.loads(machine.succeed("curl ${apiUrl}/indexes")) assert indexes["total"] == 1, "index wasn't created" with subtest("add documents"): - response = json.loads( - machine.succeed( - "curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes/${uid}/documents --data-binary @${moviesJSON}" - ) - ) - task_uid = response["taskUid"] - machine.wait_until_succeeds( - f"curl ${apiUrl}/tasks/{task_uid} | jq -e '.status == \"succeeded\"'" - ) + wait_task("curl -X POST -H 'Content-Type: application/json' ${apiUrl}/indexes/${uid}/documents --data-binary @${moviesJSON}") with subtest("search"): response = json.loads( From 5a8660f10fd1d77c8f3a14cba3f755a28b37abe3 Mon Sep 17 00:00:00 2001 From: sodiboo Date: Fri, 11 Jul 2025 22:57:29 +0200 Subject: [PATCH 2/2] nixos/meilisearch: generic settings; handle secrets better. --- nixos/doc/manual/redirects.json | 4 +- nixos/modules/services/search/meilisearch.md | 10 +- nixos/modules/services/search/meilisearch.nix | 258 +++++++++++------- nixos/modules/services/web-apps/sharkey.nix | 4 +- nixos/tests/web-apps/sharkey.nix | 4 +- 5 files changed, 174 insertions(+), 106 deletions(-) diff --git a/nixos/doc/manual/redirects.json b/nixos/doc/manual/redirects.json index fffee51d25b0..72f9ba718139 100644 --- a/nixos/doc/manual/redirects.json +++ b/nixos/doc/manual/redirects.json @@ -882,9 +882,7 @@ "index.html#module-services-meilisearch-quickstart-search" ], "module-services-meilisearch-defaults": [ - "index.html#module-services-meilisearch-defaults" - ], - "module-services-meilisearch-missing": [ + "index.html#module-services-meilisearch-defaults", "index.html#module-services-meilisearch-missing" ], "module-services-networking-yggdrasil": [ diff --git a/nixos/modules/services/search/meilisearch.md b/nixos/modules/services/search/meilisearch.md index b9f65861b1d1..db53777e063d 100644 --- a/nixos/modules/services/search/meilisearch.md +++ b/nixos/modules/services/search/meilisearch.md @@ -32,10 +32,12 @@ you first need to add documents to an index before you can search for documents. - The default nixos package doesn't come with the [dashboard](https://docs.meilisearch.com/learn/getting_started/quick_start.html#search), since the dashboard features makes some assets downloads at compile time. -- Anonymized Analytics sent to meilisearch are disabled by default. +- `no_analytics` is set to true by default. -- Default deployment is development mode. It doesn't require a secret master key. All routes are not protected and accessible. +- `http_addr` is derived from {option}`services.meilisearch.listenAddress` and {option}`services.meilisearch.listenPort`. The two sub-fields are separate because this makes it easier to consume in certain other modules. -## Missing {#module-services-meilisearch-missing} +- `db_path` is set to `/var/lib/meilisearch` by default. Upstream, the default value is equivalent to `/var/lib/meilisearch/data.ms`. -- the snapshot feature is not yet configurable from the module, it's just a matter of adding the relevant environment variables. +- `dump_dir` and `snapshot_dir` are set to `/var/lib/meilisearch/dumps` and `/var/lib/meilisearch/snapshots`, respectively. This is equivalent to the upstream defaults. + +- All other options inherit their upstream defaults. In particular, the default configuration uses `env = "development"`, which doesn't require a master key, in which case all routes are unprotected. diff --git a/nixos/modules/services/search/meilisearch.nix b/nixos/modules/services/search/meilisearch.nix index 0194a658c0d3..e8c6c3e7cdc6 100644 --- a/nixos/modules/services/search/meilisearch.nix +++ b/nixos/modules/services/search/meilisearch.nix @@ -7,19 +7,91 @@ let cfg = config.services.meilisearch; + settingsFormat = pkgs.formats.toml { }; + + # These secrets are used in the config file and can be set to paths. + secrets-with-path = + builtins.map + ( + { environment, name }: + { + inherit name environment; + setting = cfg.settings.${name}; + } + ) + [ + { + environment = "MEILI_SSL_CERT_PATH"; + name = "ssl_cert_path"; + } + { + environment = "MEILI_SSL_KEY_PATH"; + name = "ssl_key_path"; + } + { + environment = "MEILI_SSL_AUTH_PATH"; + name = "ssl_auth_path"; + } + { + environment = "MEILI_SSL_OCSP_PATH"; + name = "ssl_ocsp_path"; + } + ]; + + # We also handle `master_key` separately. + # It cannot be set to a path, so we template it. + master-key-placeholder = "@MASTER_KEY@"; + + configFile = settingsFormat.generate "config.toml" ( + builtins.removeAttrs ( + if cfg.masterKeyFile != null then + cfg.settings // { master_key = master-key-placeholder; } + else + builtins.removeAttrs cfg.settings [ "master_key" ] + ) (map (secret: secret.name) secrets-with-path) + ); + in { - meta.maintainers = with lib.maintainers; [ Br1ght0ne happysalada ]; meta.doc = ./meilisearch.md; - ###### interface + imports = [ + (lib.mkRenamedOptionModule + [ "services" "meilisearch" "environment" ] + [ "services" "meilisearch" "settings" "env" ] + ) + (lib.mkRenamedOptionModule + [ "services" "meilisearch" "logLevel" ] + [ "services" "meilisearch" "settings" "log_level" ] + ) + (lib.mkRenamedOptionModule + [ "services" "meilisearch" "noAnalytics" ] + [ "services" "meilisearch" "settings" "no_analytics" ] + ) + (lib.mkRenamedOptionModule + [ "services" "meilisearch" "maxIndexSize" ] + [ "services" "meilisearch" "settings" "max_index_size" ] + ) + (lib.mkRenamedOptionModule + [ "services" "meilisearch" "payloadSizeLimit" ] + [ "services" "meilisearch" "settings" "http_payload_size_limit" ] + ) + (lib.mkRenamedOptionModule + [ "services" "meilisearch" "dumplessUpgrade" ] + [ "services" "meilisearch" "settings" "experimental_dumpless_upgrade" ] + ) + (lib.mkRemovedOptionModule [ "services" "meilisearch" "masterKeyEnvironmentFile" ] '' + Use `services.meilisearch.masterKeyFile` instead. It does not require you to prefix the file with "MEILI_MASTER_KEY=". + If you were abusing this option to set other options, you can now configure them with `services.meilisearch.settings`. + '') + ]; options.services.meilisearch = { - enable = lib.mkEnableOption "MeiliSearch - a RESTful search API"; + enable = lib.mkEnableOption "Meilisearch - a RESTful search API"; package = lib.mkPackageOption pkgs "meilisearch" { extraDescription = '' @@ -28,106 +100,87 @@ in }; listenAddress = lib.mkOption { - description = "MeiliSearch listen address."; - default = "127.0.0.1"; + default = "localhost"; type = lib.types.str; + description = '' + The IP address that Meilisearch will listen on. + + It can also be a hostname like "localhost". If it resolves to an IPv4 and IPv6 address, Meilisearch will listen on both. + ''; }; listenPort = lib.mkOption { - description = "MeiliSearch port to listen on."; default = 7700; type = lib.types.port; + description = '' + The port that Meilisearch will listen on. + ''; }; - environment = lib.mkOption { - description = "Defines the running environment of MeiliSearch."; - default = "development"; - type = lib.types.enum [ - "development" - "production" - ]; - }; - - # TODO change this to LoadCredentials once possible - masterKeyEnvironmentFile = lib.mkOption { + masterKeyFile = lib.mkOption { description = '' Path to file which contains the master key. By doing so, all routes will be protected and will require a key to be accessed. If no master key is provided, all routes can be accessed without requiring any key. - The format is the following: - MEILI_MASTER_KEY=my_secret_key ''; default = null; - type = with lib.types; nullOr path; + type = lib.types.nullOr lib.types.path; }; - noAnalytics = lib.mkOption { + settings = lib.mkOption { description = '' - Deactivates analytics. - Analytics allow MeiliSearch to know how many users are using MeiliSearch, - which versions and which platforms are used. - This process is entirely anonymous. + Configuration settings for Meilisearch. + Look at the documentation for available options: + https://github.com/meilisearch/meilisearch/blob/main/config.toml + https://www.meilisearch.com/docs/learn/self_hosted/configure_meilisearch_at_launch#all-instance-options ''; - default = true; - type = lib.types.bool; + + default = { }; + + type = lib.types.submodule { + freeformType = settingsFormat.type; + + imports = builtins.map (secret: { + # give them proper types, just so they're easier to consume from this file + options.${secret.name} = lib.mkOption { + # but they should not show up in documentation as special in any way. + visible = false; + + type = lib.types.nullOr lib.types.path; + default = null; + }; + }) secrets-with-path; + }; }; - - logLevel = lib.mkOption { - description = '' - Defines how much detail should be present in MeiliSearch's logs. - MeiliSearch currently supports four log levels, listed in order of increasing verbosity: - - 'ERROR': only log unexpected events indicating MeiliSearch is not functioning as expected - - 'WARN:' log all unexpected events, regardless of their severity - - 'INFO:' log all events. This is the default value - - 'DEBUG': log all events and including detailed information on MeiliSearch's internal processes. - Useful when diagnosing issues and debugging - ''; - default = "INFO"; - type = lib.types.str; - }; - - maxIndexSize = lib.mkOption { - description = '' - Sets the maximum size of the index. - Value must be given in bytes or explicitly stating a base unit. - For example, the default value can be written as 107374182400, '107.7Gb', or '107374 Mb'. - Default is 100 GiB - ''; - default = "107374182400"; - type = lib.types.str; - }; - - payloadSizeLimit = lib.mkOption { - description = '' - Sets the maximum size of accepted JSON payloads. - Value must be given in bytes or explicitly stating a base unit. - For example, the default value can be written as 107374182400, '107.7Gb', or '107374 Mb'. - Default is ~ 100 MB - ''; - default = "104857600"; - type = lib.types.str; - }; - - # TODO: turn on by default when it stops being experimental - dumplessUpgrade = lib.mkOption { - default = false; - example = true; - description = '' - Whether to enable (experimental) dumpless upgrade. - - Allows upgrading from Meilisearch >=v1.12 to Meilisearch >=v1.13 without manually - dumping and importing the database. - - More information at https://www.meilisearch.com/docs/learn/update_and_migration/updating#dumpless-upgrade - ''; - type = lib.types.bool; - }; - }; - ###### implementation - config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = !cfg.settings ? master_key; + message = '' + Do not set `services.meilisearch.settings.master_key` in your configuration. + Use `services.meilisearch.masterKeyFile` instead. + ''; + } + ]; + + services.meilisearch.settings = { + # we use `listenAddress` and `listenPort` to derive the `http_addr` setting. + # this is the only setting we treat like this. + # we do this because some dependent services like Misskey/Sharkey need separate host,port for no good reason. + http_addr = "${cfg.listenAddress}:${toString cfg.listenPort}"; + + # upstream's default for `db_path` is `/var/lib/meilisearch/data.ms/`, but ours is different for no reason. + db_path = lib.mkDefault "/var/lib/meilisearch"; + # these are equivalent to the upstream defaults, because we set a working directory. + # they are only set here for consistency with `db_path`. + dump_dir = lib.mkDefault "/var/lib/meilisearch/dumps"; + snapshot_dir = lib.mkDefault "/var/lib/meilisearch/snapshots"; + + # this is intentionally different from upstream's default. + no_analytics = lib.mkDefault true; + }; warnings = lib.optional (lib.versionOlder cfg.package.version "1.12") '' Meilisearch 1.11 will be removed in NixOS 25.11. As it was the last @@ -149,24 +202,41 @@ in environment.systemPackages = [ cfg.package ]; systemd.services.meilisearch = { - description = "MeiliSearch daemon"; + description = "Meilisearch daemon"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; - environment = { - MEILI_DB_PATH = "/var/lib/meilisearch"; - MEILI_HTTP_ADDR = "${cfg.listenAddress}:${toString cfg.listenPort}"; - MEILI_NO_ANALYTICS = lib.boolToString cfg.noAnalytics; - MEILI_ENV = cfg.environment; - MEILI_DUMP_DIR = "/var/lib/meilisearch/dumps"; - MEILI_LOG_LEVEL = cfg.logLevel; - MEILI_MAX_INDEX_SIZE = cfg.maxIndexSize; - MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE = lib.boolToString cfg.dumplessUpgrade; - }; + + preStart = lib.mkMerge [ + '' + install -m 700 '${configFile}' "$RUNTIME_DIRECTORY/config.toml" + '' + (lib.mkIf (cfg.masterKeyFile != null) '' + ${lib.getExe pkgs.replace-secret} '${master-key-placeholder}' "$CREDENTIALS_DIRECTORY/master_key" "$RUNTIME_DIRECTORY/config.toml" + '') + ]; + + environment = builtins.listToAttrs ( + builtins.map (secret: { + name = secret.environment; + value = lib.mkIf (secret.setting != null) "%d/${secret.name}"; + }) secrets-with-path + ); + serviceConfig = { - ExecStart = "${cfg.package}/bin/meilisearch"; + LoadCredential = lib.mkMerge ( + [ + (lib.mkIf (cfg.masterKeyFile != null) [ "master_key:${cfg.masterKeyFile}" ]) + ] + ++ builtins.map ( + secret: lib.mkIf (secret.setting != null) [ "${secret.name}:${secret.setting}" ] + ) secrets-with-path + ); + ExecStart = "${lib.getExe cfg.package} --config-file-path \${RUNTIME_DIRECTORY}/config.toml"; DynamicUser = true; StateDirectory = "meilisearch"; - EnvironmentFile = lib.mkIf (cfg.masterKeyEnvironmentFile != null) cfg.masterKeyEnvironmentFile; + WorkingDirectory = "%S/meilisearch"; + RuntimeDirectory = "meilisearch"; + RuntimeDirectoryMode = "0700"; }; }; }; diff --git a/nixos/modules/services/web-apps/sharkey.nix b/nixos/modules/services/web-apps/sharkey.nix index 3c891ff5e892..3ff9a37784c3 100644 --- a/nixos/modules/services/web-apps/sharkey.nix +++ b/nixos/modules/services/web-apps/sharkey.nix @@ -54,7 +54,7 @@ in description = '' Whether to automatically set up a local Meilisearch instance and configure Sharkey to use it. - You need to ensure `services.meilisearch.masterKeyEnvironmentFile` is correctly configured for a working + You need to ensure `services.meilisearch.masterKeyFile` is correctly configured for a working Meilisearch setup. You also need to configure Sharkey to use an API key obtained from Meilisearch with the `MK_CONFIG_MEILISEARCH_APIKEY` environment variable, and set `services.sharkey.settings.meilisearch.index` to the created index. See https://docs.joinsharkey.org/docs/customisation/search/meilisearch/ for how to create @@ -240,7 +240,7 @@ in (mkIf cfg.setupMeilisearch { services.meilisearch = { enable = mkDefault true; - environment = mkDefault "production"; + settings.env = mkDefault "production"; }; services.sharkey.settings = { diff --git a/nixos/tests/web-apps/sharkey.nix b/nixos/tests/web-apps/sharkey.nix index b10fec68744e..f3261e612100 100644 --- a/nixos/tests/web-apps/sharkey.nix +++ b/nixos/tests/web-apps/sharkey.nix @@ -19,9 +19,7 @@ in }; }; - services.meilisearch.masterKeyEnvironmentFile = pkgs.writeText "meilisearch-key" '' - MEILI_MASTER_KEY=${meilisearchKey} - ''; + services.meilisearch.masterKeyFile = pkgs.writeText "meilisearch-key" meilisearchKey; }; testScript =