From ebaa2268539a1d7fcf81773afb0bea671a6ec340 Mon Sep 17 00:00:00 2001 From: Kim Lindberger Date: Thu, 16 Dec 2021 16:20:52 +0100 Subject: [PATCH] elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879) * elk7: 7.11.1 -> 7.16.1 * nixosTests.elk: Improve reliability and compatibility with ELK 7.x - Use comparisons in jq instead of grepping - Match for `.hits.total.value` if version >= 7, otherwise it always passes - Make curl fail if requests fails * nixos/filebeat: Add initial module and test Filebeat is an open source file harvester, mostly used to fetch logs files and feed them into logstash. This module can be used instead of journalbeat if used with `filebeat7` and configured with the `journald` input. * python3Packages.parsedmarc.tests: Fix breakage - Don't use the deprecated elasticsearch7-oss package - Improve jq query robustness and add tracing * rl-2205: Note the addition of the filebeat service * elk6: 6.8.3 -> 6.8.21 The latest version includes a fix for CVE-2021-44228. * nixos/journalbeat: Add a loose dependency on elasticsearch Avoid unnecssary back-off when elasticsearch is running on the same host. --- .../from_md/release-notes/rl-2205.section.xml | 10 +- .../manual/release-notes/rl-2205.section.md | 2 + nixos/modules/module-list.nix | 1 + nixos/modules/services/logging/filebeat.nix | 253 ++++++++++++++++++ .../modules/services/logging/journalbeat.nix | 3 +- nixos/tests/elk.nix | 90 +++++-- nixos/tests/parsedmarc/default.nix | 27 +- pkgs/development/tools/misc/kibana/6.x.nix | 8 +- pkgs/development/tools/misc/kibana/7.x.nix | 10 +- pkgs/misc/logging/beats/6.x.nix | 2 +- pkgs/misc/logging/beats/7.x.nix | 24 +- pkgs/servers/search/elasticsearch/6.x.nix | 4 +- pkgs/servers/search/elasticsearch/7.x.nix | 6 +- pkgs/servers/search/elasticsearch/plugins.nix | 44 +-- pkgs/tools/misc/logstash/6.x.nix | 4 +- pkgs/tools/misc/logstash/7.x.nix | 12 +- pkgs/top-level/aliases.nix | 1 + pkgs/top-level/all-packages.nix | 7 +- 18 files changed, 414 insertions(+), 94 deletions(-) create mode 100644 nixos/modules/services/logging/filebeat.nix diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index d02f951a03ca..2dd27649c52c 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -19,7 +19,7 @@
New Services - + aesmd, @@ -28,6 +28,14 @@ services.aesmd. + + + filebeat, + a lightweight shipper for forwarding and centralizing log + data. Available as + services.filebeat. + +
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index 11e5462b3316..595785e732a4 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin - [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable). +- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable). + ## Backward Incompatibilities {#sec-release-22.05-incompatibilities} - `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`. diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 1f826220a0f3..cb2dd530de15 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -448,6 +448,7 @@ ./services/hardware/xow.nix ./services/logging/SystemdJournal2Gelf.nix ./services/logging/awstats.nix + ./services/logging/filebeat.nix ./services/logging/fluentd.nix ./services/logging/graylog.nix ./services/logging/heartbeat.nix diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix new file mode 100644 index 000000000000..223a993c505b --- /dev/null +++ b/nixos/modules/services/logging/filebeat.nix @@ -0,0 +1,253 @@ +{ config, lib, utils, pkgs, ... }: + +let + inherit (lib) + attrValues + literalExpression + mkEnableOption + mkIf + mkOption + types; + + cfg = config.services.filebeat; + + json = pkgs.formats.json {}; +in +{ + options = { + + services.filebeat = { + + enable = mkEnableOption "filebeat"; + + package = mkOption { + type = types.package; + default = pkgs.filebeat; + defaultText = literalExpression "pkgs.filebeat"; + example = literalExpression "pkgs.filebeat7"; + description = '' + The filebeat package to use. + ''; + }; + + inputs = mkOption { + description = '' + Inputs specify how Filebeat locates and processes input data. + + This is like services.filebeat.settings.filebeat.inputs, + but structured as an attribute set. This has the benefit + that multiple NixOS modules can contribute settings to a + single filebeat input. + + An input type can be specified multiple times by choosing a + different <name> for each, but setting + + to the same value. + + See . + ''; + default = {}; + type = types.attrsOf (types.submodule ({ name, ... }: { + freeformType = json.type; + options = { + type = mkOption { + type = types.str; + default = name; + description = '' + The input type. + + Look for the value after type: on + the individual input pages linked from + . + ''; + }; + }; + })); + example = literalExpression '' + { + journald.id = "everything"; # Only for filebeat7 + log = { + enabled = true; + paths = [ + "/var/log/*.log" + ]; + }; + }; + ''; + }; + + modules = mkOption { + description = '' + Filebeat modules provide a quick way to get started + processing common log formats. They contain default + configurations, Elasticsearch ingest pipeline definitions, + and Kibana dashboards to help you implement and deploy a log + monitoring solution. + + This is like services.filebeat.settings.filebeat.modules, + but structured as an attribute set. This has the benefit + that multiple NixOS modules can contribute settings to a + single filebeat module. + + A module can be specified multiple times by choosing a + different <name> for each, but setting + + to the same value. + + See . + ''; + default = {}; + type = types.attrsOf (types.submodule ({ name, ... }: { + freeformType = json.type; + options = { + module = mkOption { + type = types.str; + default = name; + description = '' + The name of the module. + + Look for the value after module: on + the individual input pages linked from + . + ''; + }; + }; + })); + example = literalExpression '' + { + nginx = { + access = { + enabled = true; + var.paths = [ "/path/to/log/nginx/access.log*" ]; + }; + error = { + enabled = true; + var.paths = [ "/path/to/log/nginx/error.log*" ]; + }; + }; + }; + ''; + }; + + settings = mkOption { + type = types.submodule { + freeformType = json.type; + + options = { + + output.elasticsearch.hosts = mkOption { + type = with types; listOf str; + default = [ "127.0.0.1:9200" ]; + example = [ "myEShost:9200" ]; + description = '' + The list of Elasticsearch nodes to connect to. + + The events are distributed to these nodes in round + robin order. If one node becomes unreachable, the + event is automatically sent to another node. Each + Elasticsearch node can be defined as a URL or + IP:PORT. For example: + http://192.15.3.2, + https://es.found.io:9230 or + 192.24.3.2:9300. If no port is + specified, 9200 is used. + ''; + }; + + filebeat = { + inputs = mkOption { + type = types.listOf json.type; + default = []; + internal = true; + description = '' + Inputs specify how Filebeat locates and processes + input data. Use instead. + + See . + ''; + }; + modules = mkOption { + type = types.listOf json.type; + default = []; + internal = true; + description = '' + Filebeat modules provide a quick way to get started + processing common log formats. They contain default + configurations, Elasticsearch ingest pipeline + definitions, and Kibana dashboards to help you + implement and deploy a log monitoring solution. + + Use instead. + + See . + ''; + }; + }; + }; + }; + default = {}; + example = literalExpression '' + { + settings = { + output.elasticsearch = { + hosts = [ "myEShost:9200" ]; + username = "filebeat_internal"; + password = { _secret = "/var/keys/elasticsearch_password"; }; + }; + logging.level = "info"; + }; + }; + ''; + + description = '' + Configuration for filebeat. See + + for supported values. + + Options containing secret data should be set to an attribute + set containing the attribute _secret - a + string pointing to a file containing the value the option + should be set to. See the example to get a better picture of + this: in the resulting + filebeat.yml file, the + output.elasticsearch.password + key will be set to the contents of the + /var/keys/elasticsearch_password file. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + + services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs; + services.filebeat.settings.filebeat.modules = attrValues cfg.modules; + + systemd.services.filebeat = { + description = "Filebeat log shipper"; + wantedBy = [ "multi-user.target" ]; + wants = [ "elasticsearch.service" ]; + after = [ "elasticsearch.service" ]; + serviceConfig = { + ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" '' + set -euo pipefail + + umask u=rwx,g=,o= + + ${utils.genJqSecretsReplacementSnippet + cfg.settings + "/var/lib/filebeat/filebeat.yml" + } + ''; + ExecStart = '' + ${cfg.package}/bin/filebeat -e \ + -c "/var/lib/filebeat/filebeat.yml" \ + --path.data "/var/lib/filebeat" + ''; + Restart = "always"; + StateDirectory = "filebeat"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix index 2d98598c1bee..48b02142cb4c 100644 --- a/nixos/modules/services/logging/journalbeat.nix +++ b/nixos/modules/services/logging/journalbeat.nix @@ -28,7 +28,6 @@ in type = types.package; default = pkgs.journalbeat; defaultText = literalExpression "pkgs.journalbeat"; - example = literalExpression "pkgs.journalbeat7"; description = '' The journalbeat package to use ''; @@ -89,6 +88,8 @@ in systemd.services.journalbeat = { description = "Journalbeat log shipper"; wantedBy = [ "multi-user.target" ]; + wants = [ "elasticsearch.service" ]; + after = [ "elasticsearch.service" ]; preStart = '' mkdir -p ${cfg.stateDir}/data mkdir -p ${cfg.stateDir}/logs diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index ae746d7e1f03..f42be00f23b8 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -40,9 +40,8 @@ let services = { - journalbeat = let lt6 = builtins.compareVersions - elk.journalbeat.version "6" < 0; in { - enable = true; + journalbeat = { + enable = elk ? journalbeat; package = elk.journalbeat; extraConfig = pkgs.lib.mkOptionDefault ('' logging: @@ -51,14 +50,29 @@ let metrics.enabled: false output.elasticsearch: hosts: [ "127.0.0.1:9200" ] - ${pkgs.lib.optionalString lt6 "template.enabled: false"} - '' + pkgs.lib.optionalString (!lt6) '' journalbeat.inputs: - paths: [] seek: cursor ''); }; + filebeat = { + enable = elk ? filebeat; + package = elk.filebeat; + inputs.journald.id = "everything"; + + inputs.log = { + enabled = true; + paths = [ + "/var/lib/filebeat/test" + ]; + }; + + settings = { + logging.level = "info"; + }; + }; + metricbeat = { enable = true; package = elk.metricbeat; @@ -142,27 +156,43 @@ let }; passthru.elkPackages = elk; - testScript = '' + testScript = + let + valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value"; + in '' import json - def total_hits(message): + def expect_hits(message): dictionary = {"query": {"match": {"message": message}}} return ( - "curl --silent --show-error '${esUrl}/_search' " + "curl --silent --show-error --fail-with-body '${esUrl}/_search' " + "-H 'Content-Type: application/json' " + "-d '{}' ".format(json.dumps(dictionary)) - + "| jq .hits.total" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" + ) + + + def expect_no_hits(message): + dictionary = {"query": {"match": {"message": message}}} + return ( + "curl --silent --show-error --fail-with-body '${esUrl}/_search' " + + "-H 'Content-Type: application/json' " + + "-d '{}' ".format(json.dumps(dictionary)) + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'" ) def has_metricbeat(): dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}} return ( - "curl --silent --show-error '${esUrl}/_search' " + "curl --silent --show-error --fail-with-body '${esUrl}/_search' " + "-H 'Content-Type: application/json' " + "-d '{}' ".format(json.dumps(dictionary)) - + "| jq '.hits.total > 0'" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) @@ -178,7 +208,8 @@ let # TODO: extend this test with multiple elasticsearch nodes # and see if the status turns "green". one.wait_until_succeeds( - "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red" + "curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'" + + " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'" ) with subtest("Perform some simple logstash tests"): @@ -189,33 +220,50 @@ let with subtest("Kibana is healthy"): one.wait_for_unit("kibana.service") one.wait_until_succeeds( - "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green" + "curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'" + + " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'" ) with subtest("Metricbeat is running"): one.wait_for_unit("metricbeat.service") with subtest("Metricbeat metrics arrive in elasticsearch"): - one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'") + one.wait_until_succeeds(has_metricbeat()) with subtest("Logstash messages arive in elasticsearch"): - one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0") - one.wait_until_succeeds(total_hits("dragons") + " | grep 0") + one.wait_until_succeeds(expect_hits("flowers")) + one.wait_until_succeeds(expect_no_hits("dragons")) + '' + lib.optionalString (elk ? journalbeat) '' with subtest( "A message logged to the journal is ingested by elasticsearch via journalbeat" ): one.wait_for_unit("journalbeat.service") one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat") one.wait_until_succeeds( - total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0" + expect_hits("Supercalifragilisticexpialidocious") ) - + '' + lib.optionalString (elk ? filebeat) '' + with subtest( + "A message logged to the journal is ingested by elasticsearch via filebeat" + ): + one.wait_for_unit("filebeat.service") + one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat") + one.wait_until_succeeds( + expect_hits("Superdupercalifragilisticexpialidocious") + ) + one.execute( + "echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test" + ) + one.wait_until_succeeds( + expect_hits("SuperdupercalifragilisticexpialidociousIndeed") + ) + '' + '' with subtest("Elasticsearch-curator works"): one.systemctl("stop logstash") one.systemctl("start elasticsearch-curator") one.wait_until_succeeds( - '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^' + '! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^' ) ''; }) { inherit pkgs system; }; @@ -235,7 +283,7 @@ in { # elasticsearch = pkgs.elasticsearch7-oss; # logstash = pkgs.logstash7-oss; # kibana = pkgs.kibana7-oss; - # journalbeat = pkgs.journalbeat7; + # filebeat = pkgs.filebeat7; # metricbeat = pkgs.metricbeat7; # }; unfree = lib.dontRecurseIntoAttrs { @@ -250,7 +298,7 @@ in { elasticsearch = pkgs.elasticsearch7; logstash = pkgs.logstash7; kibana = pkgs.kibana7; - journalbeat = pkgs.journalbeat7; + filebeat = pkgs.filebeat7; metricbeat = pkgs.metricbeat7; }; }; diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix index d838d3b6a39c..50b977723e9c 100644 --- a/nixos/tests/parsedmarc/default.nix +++ b/nixos/tests/parsedmarc/default.nix @@ -4,6 +4,7 @@ { pkgs, ... }@args: let inherit (import ../../lib/testing-python.nix args) makeTest; + inherit (pkgs) lib; dmarcTestReport = builtins.fetchurl { name = "dmarc-test-report"; @@ -54,7 +55,7 @@ in localMail = makeTest { name = "parsedmarc-local-mail"; - meta = with pkgs.lib.maintainers; { + meta = with lib.maintainers; { maintainers = [ talyz ]; }; @@ -83,7 +84,7 @@ in }; }; - services.elasticsearch.package = pkgs.elasticsearch7-oss; + services.elasticsearch.package = pkgs.elasticsearch-oss; environment.systemPackages = [ (sendEmail "dmarc@localhost") @@ -94,6 +95,7 @@ in testScript = { nodes }: let esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value"; in '' parsedmarc.start() parsedmarc.wait_for_unit("postfix.service") @@ -104,11 +106,15 @@ in ) parsedmarc.fail( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) parsedmarc.succeed("send-email") parsedmarc.wait_until_succeeds( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) ''; }; @@ -121,7 +127,7 @@ in in makeTest { name = "parsedmarc-external-mail"; - meta = with pkgs.lib.maintainers; { + meta = with lib.maintainers; { maintainers = [ talyz ]; }; @@ -153,7 +159,7 @@ in }; }; - services.elasticsearch.package = pkgs.elasticsearch7-oss; + services.elasticsearch.package = pkgs.elasticsearch-oss; environment.systemPackages = [ pkgs.jq @@ -201,6 +207,7 @@ in testScript = { nodes }: let esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value"; in '' mail.start() mail.wait_for_unit("postfix.service") @@ -213,11 +220,15 @@ in ) parsedmarc.fail( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) mail.succeed("send-email") parsedmarc.wait_until_succeeds( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) ''; }; diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix index c728f7375436..cd81975ee445 100644 --- a/pkgs/development/tools/misc/kibana/6.x.nix +++ b/pkgs/development/tools/misc/kibana/6.x.nix @@ -18,12 +18,12 @@ let shas = if enableUnfree then { - x86_64-linux = "1xwklhqxk5rmdrgy2simwvijzq29kyq5w2w3hy53xh2i1zlnyvq3"; - x86_64-darwin = "1qpdn28mrpggd55khzqqld6r89l0hb870rigxcw2i8p2yx3jv106"; + x86_64-linux = "1a501lavxhckb3l93sbrbqyshicwkk6p89frry4x8p037xcfpy0x"; + x86_64-darwin = "0zm45af30shhcg3mdhcma6rms1hyrx62rm5jzwnz9kxv4d30skbw"; } else { - x86_64-linux = "1wpnwal2rq5v2bsp5qil9j6dplif7ql5394sy4ia5ghp2fzifxmf"; - x86_64-darwin = "12z8i0wbw10c097glbpdy350p0h3957433f51qfx2p0ghgkzkhzv"; + x86_64-linux = "0wfdipf21apyily7mvlqgyc7m5jpr96zgrryzwa854z3xb2vw8zg"; + x86_64-darwin = "1nklfx4yz6hsxlljvnvwjy7pncv9mzngl84710xad5jlyras3sdj"; }; in stdenv.mkDerivation rec { diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix index 7a2e0d1d3647..455f95a69eb0 100644 --- a/pkgs/development/tools/misc/kibana/7.x.nix +++ b/pkgs/development/tools/misc/kibana/7.x.nix @@ -4,23 +4,23 @@ , stdenv , makeWrapper , fetchurl -, nodejs-14_x +, nodejs-16_x , coreutils , which }: with lib; let - nodejs = nodejs-14_x; + nodejs = nodejs-16_x; inherit (builtins) elemAt; info = splitString "-" stdenv.hostPlatform.system; arch = elemAt info 0; plat = elemAt info 1; shas = { - x86_64-linux = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1"; - x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc"; - aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8"; + x86_64-linux = "0jivwsrq31n0qfznrsjfsn65sg3wpbd990afn2wzjnj4drq7plz6"; + x86_64-darwin = "02483aqzrccq1x6rwznmcazijdd46yxj9vnbihnvp2xyp3w9as45"; + aarch64-linux = "0iw155gkkl1hshc80lfj95rssg039ig21wz1l3srmmf2x4f934s9"; }; in stdenv.mkDerivation rec { diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix index 1808197498b5..f5e31924791b 100644 --- a/pkgs/misc/logging/beats/6.x.nix +++ b/pkgs/misc/logging/beats/6.x.nix @@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8"; + sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7"; }; goPackagePath = "github.com/elastic/beats"; diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix index b8b82ed4b308..9f5e550d9720 100644 --- a/pkgs/misc/logging/beats/7.x.nix +++ b/pkgs/misc/logging/beats/7.x.nix @@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8"; + sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI="; }; - vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c"; + vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk="; subPackages = [ package ]; @@ -24,7 +24,14 @@ let beat = package: extraArgs: buildGoModule (rec { } // extraArgs); in rec { - filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; }; + filebeat7 = beat "filebeat" { + meta.description = "Lightweight shipper for logfiles"; + buildInputs = [ systemd ]; + tags = [ "withjournald" ]; + postFixup = '' + patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat" + ''; + }; heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; }; metricbeat7 = beat "metricbeat" { meta.description = "Lightweight shipper for metrics"; @@ -47,15 +54,4 @@ rec { PostgreSQL, Redis or Thrift and correlate the messages into transactions. ''; }; - journalbeat7 = beat "journalbeat" { - meta.description = '' - Journalbeat is an open source data collector to read and forward - journal entries from Linuxes with systemd. - ''; - buildInputs = [ systemd.dev ]; - postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in - '' - patchelf --set-rpath ${libPath} "$out/bin/journalbeat" - ''; - }; } diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix index 4b92592f65df..56f0779c833a 100644 --- a/pkgs/servers/search/elasticsearch/6.x.nix +++ b/pkgs/servers/search/elasticsearch/6.x.nix @@ -19,8 +19,8 @@ stdenv.mkDerivation (rec { url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz"; sha256 = if enableUnfree - then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42" - else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39"; + then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd" + else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm"; }; patches = [ ./es-home-6.x.patch ]; diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix index c254b733837b..592cc947a42e 100644 --- a/pkgs/servers/search/elasticsearch/7.x.nix +++ b/pkgs/servers/search/elasticsearch/7.x.nix @@ -18,9 +18,9 @@ let plat = elemAt info 1; shas = { - x86_64-linux = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10"; - x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3"; - aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy"; + x86_64-linux = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c"; + x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs"; + aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq"; }; in stdenv.mkDerivation rec { diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index f71d7b9cc76c..03bb24d9a390 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -38,8 +38,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy" - else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv" + if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p" + else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -55,8 +55,8 @@ in src = fetchurl { url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip"; sha256 = - if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3" - else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53" + if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir" + else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -72,8 +72,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855" - else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37" + if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x" + else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -89,8 +89,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6" - else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im" + if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii" + else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -106,8 +106,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs" - else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll" + if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4" + else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -123,8 +123,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip"; sha256 = - if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs" - else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x" + if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml" + else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -140,8 +140,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip"; sha256 = - if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7" - else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk" + if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1" + else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -157,24 +157,24 @@ in pluginName = "search-guard"; version = # https://docs.search-guard.com/latest/search-guard-versions - if esVersion == "7.11.1" then "${esVersion}-50.0.0" - else if esVersion == "6.8.3" then "${esVersion}-25.5" + if esVersion == "7.16.1" then "${esVersion}-52.5.0" + else if esVersion == "6.8.21" then "${esVersion}-25.6" else throw "unsupported version ${esVersion} for plugin ${pluginName}"; src = - if esVersion == "7.11.1" then + if esVersion == "7.16.1" then fetchurl { url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip"; - sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm"; + sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb"; } - else if esVersion == "6.8.3" then + else if esVersion == "6.8.21" then fetchurl { - url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip"; - sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb"; + url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip"; + sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2"; } else throw "unsupported version ${version} for plugin ${pluginName}"; meta = with lib; { homepage = "https://search-guard.com"; - description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. "; + description = "Elasticsearch plugin that offers encryption, authentication, and authorisation."; license = licenses.asl20; }; }; diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix index b35e5a4aea43..0b3e17818dcd 100644 --- a/pkgs/tools/misc/logstash/6.x.nix +++ b/pkgs/tools/misc/logstash/6.x.nix @@ -17,8 +17,8 @@ let this = stdenv.mkDerivation rec { url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz"; sha256 = if enableUnfree - then "00pwi7clgdflzzg15bh3y30gzikvvy7p5fl88fww7xhhy47q8053" - else "0spxgqsyh72n0l0xh6rljp0lbqz46xmr02sqz25ybycr4qkxdhgk"; + then "0hij1byw5b3xmk3vshr9p7gxwbjrywr7ylps05ydc2dmnz8q2a79" + else "1fa236pvhj7spys54nqi3k64rwzf6zi6gaccmqg4p4sh92jzsybv"; }; dontBuild = true; diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix index 1e69fbc976d9..636c380817ce 100644 --- a/pkgs/tools/misc/logstash/7.x.nix +++ b/pkgs/tools/misc/logstash/7.x.nix @@ -17,14 +17,14 @@ let shas = if enableUnfree then { - x86_64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p"; - x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm"; - aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8"; + x86_64-linux = "1vm53alq9q1qy2jcsjg9z339xrkac5r9qqpdafp53ny4zsv1n7vj"; + x86_64-darwin = "0hhjyl04h3gd66rdk22272rj419br4v2i59lyrmaj6hmnsqbv968"; + aarch64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p"; } else { - x86_64-linux = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3"; - x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs"; - aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2"; + x86_64-linux = "1f3659vcgczm7v03q3fvsmp1ndp6wm3i7r2b2vbl4xq7hf9v7azk"; + x86_64-darwin = "10zw9qc0lc0x9in0nkxc1aiazhyd69l8sya2ni46ivyyjwf0sqsn"; + aarch64-linux = "1czhgmky2zf3mqykn5ww4257yfhd36mi4x6dq569ymly83pivf8v"; }; this = stdenv.mkDerivation rec { version = elk7Version; diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 222c6ac0474f..672fdf21b025 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -384,6 +384,7 @@ mapAliases ({ jbuilder = dune_1; # added 2018-09-09 jikes = throw "jikes was deprecated on 2019-10-07: abandoned by upstream"; joseki = apache-jena-fuseki; # added 2016-02-28 + journalbeat7 = throw "journalbeat has been removed upstream. Use filebeat with the journald input instead."; jvmci8 = throw "graalvm8 and its tools were deprecated in favor of graalvm8-ce"; # added 2021-10-15 json_glib = json-glib; # added 2018-02-25 kafkacat = kcat; # added 2021-10-07 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 8d98f53a3318..9c117ca4d712 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2140,8 +2140,7 @@ with pkgs; filebeat7 heartbeat7 metricbeat7 - packetbeat7 - journalbeat7; + packetbeat7; filebeat = filebeat6; heartbeat = heartbeat6; @@ -5000,8 +4999,8 @@ with pkgs; # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. # When updating make sure to update all plugins or they will break! - elk6Version = "6.8.3"; - elk7Version = "7.11.1"; + elk6Version = "6.8.21"; + elk7Version = "7.16.1"; elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { util-linux = util-linuxMinimal;