diff --git a/nixos/doc/manual/release-notes/rl-2305.section.md b/nixos/doc/manual/release-notes/rl-2305.section.md index 9ccb36bcbf4b..460d03b6c6de 100644 --- a/nixos/doc/manual/release-notes/rl-2305.section.md +++ b/nixos/doc/manual/release-notes/rl-2305.section.md @@ -36,6 +36,8 @@ In addition to numerous new and upgraded packages, this release has the followin - [imaginary](https://github.com/h2non/imaginary), a microservice for high-level image processing that Nextcloud can use to generate previews. Available as [services.imaginary](#opt-services.imaginary.enable). +- [opensearch](https://opensearch.org), a search server alternative to Elasticsearch. Available as [services.opensearch](options.html#opt-services.opensearch.enable). + - [goeland](https://github.com/slurdge/goeland), an alternative to rss2email written in golang with many filters. Available as [services.goeland](#opt-services.goeland.enable). - [atuin](https://github.com/ellie/atuin), a sync server for shell history. Available as [services.atuin](#opt-services.atuin.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 74651efc0c5a..9ce78c9eb124 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1048,6 +1048,7 @@ ./services/search/hound.nix ./services/search/kibana.nix ./services/search/meilisearch.nix + ./services/search/opensearch.nix ./services/search/solr.nix ./services/security/aesmd.nix ./services/security/certmgr.nix diff --git a/nixos/modules/services/search/opensearch.nix b/nixos/modules/services/search/opensearch.nix new file mode 100644 index 000000000000..df699b56747e --- /dev/null +++ b/nixos/modules/services/search/opensearch.nix @@ -0,0 +1,244 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.opensearch; + + settingsFormat = pkgs.formats.yaml {}; + + configDir = cfg.dataDir + "/config"; + + usingDefaultDataDir = cfg.dataDir == "/var/lib/opensearch"; + usingDefaultUserAndGroup = cfg.user == "opensearch" && cfg.group == "opensearch"; + + opensearchYml = settingsFormat.generate "opensearch.yml" cfg.settings; + + loggingConfigFilename = "log4j2.properties"; + loggingConfigFile = pkgs.writeTextFile { + name = loggingConfigFilename; + text = cfg.logging; + }; +in +{ + + options.services.opensearch = { + enable = mkEnableOption (lib.mdDoc "OpenSearch"); + + package = lib.mkPackageOptionMD pkgs "OpenSearch" { + default = [ "opensearch" ]; + }; + + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = settingsFormat.type; + + options."network.host" = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = lib.mdDoc '' + Which port this service should listen on. + ''; + }; + + options."cluster.name" = lib.mkOption { + type = lib.types.str; + default = "opensearch"; + description = lib.mdDoc '' + The name of the cluster. + ''; + }; + + options."discovery.type" = lib.mkOption { + type = lib.types.str; + default = "single-node"; + description = lib.mdDoc '' + The type of discovery to use. + ''; + }; + + options."http.port" = lib.mkOption { + type = lib.types.port; + default = 9200; + description = lib.mdDoc '' + The port to listen on for HTTP traffic. + ''; + }; + + options."transport.port" = lib.mkOption { + type = lib.types.port; + default = 9300; + description = lib.mdDoc '' + The port to listen on for transport traffic. + ''; + }; + }; + + default = {}; + + description = lib.mdDoc '' + OpenSearch configuration. + ''; + }; + + logging = lib.mkOption { + description = lib.mdDoc "opensearch logging configuration."; + + default = '' + logger.action.name = org.opensearch.action + logger.action.level = info + + appender.console.type = Console + appender.console.name = console + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + + rootLogger.level = info + rootLogger.appenderRef.console.ref = console + ''; + type = types.str; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/opensearch"; + apply = converge (removeSuffix "/"); + description = lib.mdDoc '' + Data directory for OpenSearch. If you change this, you need to + manually create the directory. You also need to create the + `opensearch` user and group, or change + [](#opt-services.opensearch.user) and + [](#opt-services.opensearch.group) to existing ones with + access to the directory. + ''; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "opensearch"; + description = lib.mdDoc '' + The user OpenSearch runs as. Should be left at default unless + you have very specific needs. + ''; + }; + + group = lib.mkOption { + type = lib.types.str; + default = "opensearch"; + description = lib.mdDoc '' + The group OpenSearch runs as. Should be left at default unless + you have very specific needs. + ''; + }; + + extraCmdLineOptions = lib.mkOption { + description = lib.mdDoc "Extra command line options for the OpenSearch launcher."; + default = [ ]; + type = lib.types.listOf lib.types.str; + }; + + extraJavaOptions = lib.mkOption { + description = lib.mdDoc "Extra command line options for Java."; + default = [ ]; + type = lib.types.listOf lib.types.str; + example = [ "-Djava.net.preferIPv4Stack=true" ]; + }; + + restartIfChanged = lib.mkOption { + type = lib.types.bool; + description = lib.mdDoc '' + Automatically restart the service on config change. + This can be set to false to defer restarts on a server or cluster. + Please consider the security implications of inadvertently running an older version, + and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option. + ''; + default = true; + }; + }; + + config = mkIf cfg.enable { + systemd.services.opensearch = { + description = "OpenSearch Daemon"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ pkgs.inetutils ]; + inherit (cfg) restartIfChanged; + environment = { + OPENSEARCH_HOME = cfg.dataDir; + OPENSEARCH_JAVA_OPTS = toString cfg.extraJavaOptions; + OPENSEARCH_PATH_CONF = configDir; + }; + serviceConfig = { + ExecStartPre = + let + startPreFullPrivileges = '' + set -o errexit -o pipefail -o nounset -o errtrace + shopt -s inherit_errexit + '' + (optionalString (!config.boot.isContainer) '' + # Only set vm.max_map_count if lower than ES required minimum + # This avoids conflict if configured via boot.kernel.sysctl + if [ $(${pkgs.procps}/bin/sysctl -n vm.max_map_count) -lt 262144 ]; then + ${pkgs.procps}/bin/sysctl -w vm.max_map_count=262144 + fi + ''); + startPreUnprivileged = '' + set -o errexit -o pipefail -o nounset -o errtrace + shopt -s inherit_errexit + + # Install plugins + ln -sfT ${cfg.package}/lib ${cfg.dataDir}/lib + ln -sfT ${cfg.package}/modules ${cfg.dataDir}/modules + + # opensearch needs to create the opensearch.keystore in the config directory + # so this directory needs to be writable. + mkdir -p ${configDir} + chmod 0700 ${configDir} + + # Note that we copy config files from the nix store instead of symbolically linking them + # because otherwise X-Pack Security will raise the following exception: + # java.security.AccessControlException: + # access denied ("java.io.FilePermission" "/var/lib/opensearch/config/opensearch.yml" "read") + + cp ${opensearchYml} ${configDir}/opensearch.yml + + # Make sure the logging configuration for old OpenSearch versions is removed: + rm -f "${configDir}/logging.yml" + cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename} + mkdir -p ${configDir}/scripts + cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options + + # redirect jvm logs to the data directory + mkdir -p ${cfg.dataDir}/logs + chmod 0700 ${cfg.dataDir}/logs + sed -e '#logs/gc.log#${cfg.dataDir}/logs/gc.log#' -i ${configDir}/jvm.options + ''; + in [ + "+${pkgs.writeShellScript "opensearch-start-pre-full-privileges" startPreFullPrivileges}" + "${pkgs.writeShellScript "opensearch-start-pre-unprivileged" startPreUnprivileged}" + ]; + ExecStartPost = pkgs.writeShellScript "opensearch-start-post" '' + set -o errexit -o pipefail -o nounset -o errtrace + shopt -s inherit_errexit + + # Make sure opensearch is up and running before dependents + # are started + while ! ${pkgs.curl}/bin/curl -sS -f http://${cfg.settings."network.host"}:${toString cfg.settings."http.port"} 2>/dev/null; do + sleep 1 + done + ''; + ExecStart = "${cfg.package}/bin/opensearch ${toString cfg.extraCmdLineOptions}"; + User = cfg.user; + Group = cfg.group; + LimitNOFILE = "1024000"; + Restart = "always"; + TimeoutStartSec = "infinity"; + DynamicUser = usingDefaultUserAndGroup && usingDefaultDataDir; + } // (optionalAttrs (usingDefaultDataDir) { + StateDirectory = "opensearch"; + StateDirectoryMode = "0700"; + }); + }; + + environment.systemPackages = [ cfg.package ]; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index ced2687ed9a4..86dd096afc0b 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -490,6 +490,7 @@ in { ombi = handleTest ./ombi.nix {}; openarena = handleTest ./openarena.nix {}; openldap = handleTest ./openldap.nix {}; + opensearch = discoverTests (import ./opensearch.nix); openresty-lua = handleTest ./openresty-lua.nix {}; opensmtpd = handleTest ./opensmtpd.nix {}; opensmtpd-rspamd = handleTest ./opensmtpd-rspamd.nix {}; diff --git a/nixos/tests/opensearch.nix b/nixos/tests/opensearch.nix new file mode 100644 index 000000000000..c0caf950cb9c --- /dev/null +++ b/nixos/tests/opensearch.nix @@ -0,0 +1,52 @@ +let + opensearchTest = + import ./make-test-python.nix ( + { pkgs, lib, extraSettings ? {} }: { + name = "opensearch"; + meta.maintainers = with pkgs.lib.maintainers; [ shyim ]; + + nodes.machine = lib.mkMerge [ + { + virtualisation.memorySize = 2048; + services.opensearch.enable = true; + } + extraSettings + ]; + + testScript = '' + machine.start() + machine.wait_for_unit("opensearch.service") + machine.wait_for_open_port(9200) + + machine.succeed( + "curl --fail localhost:9200" + ) + ''; + }); +in +{ + opensearch = opensearchTest {}; + opensearchCustomPathAndUser = opensearchTest { + extraSettings = { + services.opensearch.dataDir = "/var/opensearch_test"; + services.opensearch.user = "open_search"; + services.opensearch.group = "open_search"; + system.activationScripts.createDirectory = { + text = '' + mkdir -p "/var/opensearch_test" + chown open_search:open_search /var/opensearch_test + chmod 0700 /var/opensearch_test + ''; + deps = [ "users" "groups" ]; + }; + users = { + groups.open_search = {}; + users.open_search = { + description = "OpenSearch daemon user"; + group = "open_search"; + isSystemUser = true; + }; + }; + }; + }; +} diff --git a/pkgs/servers/search/opensearch/default.nix b/pkgs/servers/search/opensearch/default.nix new file mode 100644 index 000000000000..b8274d894b27 --- /dev/null +++ b/pkgs/servers/search/opensearch/default.nix @@ -0,0 +1,54 @@ +{ lib +, stdenvNoCC +, fetchurl +, makeWrapper +, jre_headless +, util-linux +, gnugrep +, coreutils +, autoPatchelfHook +, zlib +, nixosTests +}: + +stdenvNoCC.mkDerivation rec { + pname = "opensearch"; + version = "2.5.0"; + + src = fetchurl { + url = "https://artifacts.opensearch.org/releases/bundle/opensearch/${version}/opensearch-${version}-linux-x64.tar.gz"; + hash = "sha256-WPD5StVBb/hK+kP/1wkQQBKRQma/uaP+8ULeIFUBL1U="; + }; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ jre_headless util-linux ]; + patches = [./opensearch-home-fix.patch ]; + + installPhase = '' + runHook preInstall + + mkdir -p $out + cp -R bin config lib modules plugins $out + + substituteInPlace $out/bin/opensearch \ + --replace 'bin/opensearch-keystore' "$out/bin/opensearch-keystore" + + wrapProgram $out/bin/opensearch \ + --prefix PATH : "${lib.makeBinPath [ util-linux gnugrep coreutils ]}" \ + --set JAVA_HOME "${jre_headless}" + + wrapProgram $out/bin/opensearch-plugin --set JAVA_HOME "${jre_headless}" + + runHook postInstall + ''; + + passthru.tests = nixosTests.opensearch; + + meta = { + description = "Open Source, Distributed, RESTful Search Engine"; + homepage = "https://github.com/opensearch-project/OpenSearch"; + license = lib.licenses.asl20; + platforms = lib.platforms.unix; + maintainers = with lib.maintainers; [ shyim ]; + }; +} diff --git a/pkgs/servers/search/opensearch/opensearch-home-fix.patch b/pkgs/servers/search/opensearch/opensearch-home-fix.patch new file mode 100644 index 000000000000..25d0130d6d9b --- /dev/null +++ b/pkgs/servers/search/opensearch/opensearch-home-fix.patch @@ -0,0 +1,26 @@ +diff -Naur a/bin/opensearch-env b/bin/opensearch-env +--- a/bin/opensearch-env 2017-12-12 13:31:51.000000000 +0100 ++++ b/bin/opensearch-env 2017-12-18 19:51:12.282809695 +0100 +@@ -19,18 +19,10 @@ + fi + done + +-# determine OpenSearch home; to do this, we strip from the path until we find +-# bin, and then strip bin (there is an assumption here that there is no nested +-# directory under bin also named bin) +-OPENSEARCH_HOME=`dirname "$SCRIPT"` +- +-# now make OPENSEARCH_HOME absolute +-OPENSEARCH_HOME=`cd "$OPENSEARCH_HOME"; pwd` +- +-while [ "`basename "$OPENSEARCH_HOME"`" != "bin" ]; do +- OPENSEARCH_HOME=`dirname "$OPENSEARCH_HOME"` +-done +-OPENSEARCH_HOME=`dirname "$OPENSEARCH_HOME"` ++if [ -z "$OPENSEARCH_HOME" ]; then ++ echo "You must set the OPENSEARCH_HOME var" >&2 ++ exit 1 ++fi + + # now set the classpath + OPENSEARCH_CLASSPATH="$OPENSEARCH_HOME/lib/*" \ No newline at end of file diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c6c7aa21b677..ab1d60a1ad31 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -34819,6 +34819,8 @@ with pkgs; openrct2 = callPackage ../games/openrct2 { }; + opensearch = callPackage ../servers/search/opensearch { }; + osu-lazer = callPackage ../games/osu-lazer { }; osu-lazer-bin = callPackage ../games/osu-lazer/bin.nix { };