diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index d02f951a03ca..2dd27649c52c 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -19,7 +19,7 @@
New Services - + aesmd, @@ -28,6 +28,14 @@ services.aesmd. + + + filebeat, + a lightweight shipper for forwarding and centralizing log + data. Available as + services.filebeat. + +
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index 11e5462b3316..595785e732a4 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin - [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable). +- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable). + ## Backward Incompatibilities {#sec-release-22.05-incompatibilities} - `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`. diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 1f826220a0f3..cb2dd530de15 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -448,6 +448,7 @@ ./services/hardware/xow.nix ./services/logging/SystemdJournal2Gelf.nix ./services/logging/awstats.nix + ./services/logging/filebeat.nix ./services/logging/fluentd.nix ./services/logging/graylog.nix ./services/logging/heartbeat.nix diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix new file mode 100644 index 000000000000..223a993c505b --- /dev/null +++ b/nixos/modules/services/logging/filebeat.nix @@ -0,0 +1,253 @@ +{ config, lib, utils, pkgs, ... }: + +let + inherit (lib) + attrValues + literalExpression + mkEnableOption + mkIf + mkOption + types; + + cfg = config.services.filebeat; + + json = pkgs.formats.json {}; +in +{ + options = { + + services.filebeat = { + + enable = mkEnableOption "filebeat"; + + package = mkOption { + type = types.package; + default = pkgs.filebeat; + defaultText = literalExpression "pkgs.filebeat"; + example = literalExpression "pkgs.filebeat7"; + description = '' + The filebeat package to use. + ''; + }; + + inputs = mkOption { + description = '' + Inputs specify how Filebeat locates and processes input data. + + This is like services.filebeat.settings.filebeat.inputs, + but structured as an attribute set. This has the benefit + that multiple NixOS modules can contribute settings to a + single filebeat input. + + An input type can be specified multiple times by choosing a + different <name> for each, but setting + + to the same value. + + See . + ''; + default = {}; + type = types.attrsOf (types.submodule ({ name, ... }: { + freeformType = json.type; + options = { + type = mkOption { + type = types.str; + default = name; + description = '' + The input type. + + Look for the value after type: on + the individual input pages linked from + . + ''; + }; + }; + })); + example = literalExpression '' + { + journald.id = "everything"; # Only for filebeat7 + log = { + enabled = true; + paths = [ + "/var/log/*.log" + ]; + }; + }; + ''; + }; + + modules = mkOption { + description = '' + Filebeat modules provide a quick way to get started + processing common log formats. They contain default + configurations, Elasticsearch ingest pipeline definitions, + and Kibana dashboards to help you implement and deploy a log + monitoring solution. + + This is like services.filebeat.settings.filebeat.modules, + but structured as an attribute set. This has the benefit + that multiple NixOS modules can contribute settings to a + single filebeat module. + + A module can be specified multiple times by choosing a + different <name> for each, but setting + + to the same value. + + See . + ''; + default = {}; + type = types.attrsOf (types.submodule ({ name, ... }: { + freeformType = json.type; + options = { + module = mkOption { + type = types.str; + default = name; + description = '' + The name of the module. + + Look for the value after module: on + the individual input pages linked from + . + ''; + }; + }; + })); + example = literalExpression '' + { + nginx = { + access = { + enabled = true; + var.paths = [ "/path/to/log/nginx/access.log*" ]; + }; + error = { + enabled = true; + var.paths = [ "/path/to/log/nginx/error.log*" ]; + }; + }; + }; + ''; + }; + + settings = mkOption { + type = types.submodule { + freeformType = json.type; + + options = { + + output.elasticsearch.hosts = mkOption { + type = with types; listOf str; + default = [ "127.0.0.1:9200" ]; + example = [ "myEShost:9200" ]; + description = '' + The list of Elasticsearch nodes to connect to. + + The events are distributed to these nodes in round + robin order. If one node becomes unreachable, the + event is automatically sent to another node. Each + Elasticsearch node can be defined as a URL or + IP:PORT. For example: + http://192.15.3.2, + https://es.found.io:9230 or + 192.24.3.2:9300. If no port is + specified, 9200 is used. + ''; + }; + + filebeat = { + inputs = mkOption { + type = types.listOf json.type; + default = []; + internal = true; + description = '' + Inputs specify how Filebeat locates and processes + input data. Use instead. + + See . + ''; + }; + modules = mkOption { + type = types.listOf json.type; + default = []; + internal = true; + description = '' + Filebeat modules provide a quick way to get started + processing common log formats. They contain default + configurations, Elasticsearch ingest pipeline + definitions, and Kibana dashboards to help you + implement and deploy a log monitoring solution. + + Use instead. + + See . + ''; + }; + }; + }; + }; + default = {}; + example = literalExpression '' + { + settings = { + output.elasticsearch = { + hosts = [ "myEShost:9200" ]; + username = "filebeat_internal"; + password = { _secret = "/var/keys/elasticsearch_password"; }; + }; + logging.level = "info"; + }; + }; + ''; + + description = '' + Configuration for filebeat. See + + for supported values. + + Options containing secret data should be set to an attribute + set containing the attribute _secret - a + string pointing to a file containing the value the option + should be set to. See the example to get a better picture of + this: in the resulting + filebeat.yml file, the + output.elasticsearch.password + key will be set to the contents of the + /var/keys/elasticsearch_password file. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + + services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs; + services.filebeat.settings.filebeat.modules = attrValues cfg.modules; + + systemd.services.filebeat = { + description = "Filebeat log shipper"; + wantedBy = [ "multi-user.target" ]; + wants = [ "elasticsearch.service" ]; + after = [ "elasticsearch.service" ]; + serviceConfig = { + ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" '' + set -euo pipefail + + umask u=rwx,g=,o= + + ${utils.genJqSecretsReplacementSnippet + cfg.settings + "/var/lib/filebeat/filebeat.yml" + } + ''; + ExecStart = '' + ${cfg.package}/bin/filebeat -e \ + -c "/var/lib/filebeat/filebeat.yml" \ + --path.data "/var/lib/filebeat" + ''; + Restart = "always"; + StateDirectory = "filebeat"; + }; + }; + }; +} diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix index 2d98598c1bee..48b02142cb4c 100644 --- a/nixos/modules/services/logging/journalbeat.nix +++ b/nixos/modules/services/logging/journalbeat.nix @@ -28,7 +28,6 @@ in type = types.package; default = pkgs.journalbeat; defaultText = literalExpression "pkgs.journalbeat"; - example = literalExpression "pkgs.journalbeat7"; description = '' The journalbeat package to use ''; @@ -89,6 +88,8 @@ in systemd.services.journalbeat = { description = "Journalbeat log shipper"; wantedBy = [ "multi-user.target" ]; + wants = [ "elasticsearch.service" ]; + after = [ "elasticsearch.service" ]; preStart = '' mkdir -p ${cfg.stateDir}/data mkdir -p ${cfg.stateDir}/logs diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index ae746d7e1f03..f42be00f23b8 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -40,9 +40,8 @@ let services = { - journalbeat = let lt6 = builtins.compareVersions - elk.journalbeat.version "6" < 0; in { - enable = true; + journalbeat = { + enable = elk ? journalbeat; package = elk.journalbeat; extraConfig = pkgs.lib.mkOptionDefault ('' logging: @@ -51,14 +50,29 @@ let metrics.enabled: false output.elasticsearch: hosts: [ "127.0.0.1:9200" ] - ${pkgs.lib.optionalString lt6 "template.enabled: false"} - '' + pkgs.lib.optionalString (!lt6) '' journalbeat.inputs: - paths: [] seek: cursor ''); }; + filebeat = { + enable = elk ? filebeat; + package = elk.filebeat; + inputs.journald.id = "everything"; + + inputs.log = { + enabled = true; + paths = [ + "/var/lib/filebeat/test" + ]; + }; + + settings = { + logging.level = "info"; + }; + }; + metricbeat = { enable = true; package = elk.metricbeat; @@ -142,27 +156,43 @@ let }; passthru.elkPackages = elk; - testScript = '' + testScript = + let + valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value"; + in '' import json - def total_hits(message): + def expect_hits(message): dictionary = {"query": {"match": {"message": message}}} return ( - "curl --silent --show-error '${esUrl}/_search' " + "curl --silent --show-error --fail-with-body '${esUrl}/_search' " + "-H 'Content-Type: application/json' " + "-d '{}' ".format(json.dumps(dictionary)) - + "| jq .hits.total" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" + ) + + + def expect_no_hits(message): + dictionary = {"query": {"match": {"message": message}}} + return ( + "curl --silent --show-error --fail-with-body '${esUrl}/_search' " + + "-H 'Content-Type: application/json' " + + "-d '{}' ".format(json.dumps(dictionary)) + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'" ) def has_metricbeat(): dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}} return ( - "curl --silent --show-error '${esUrl}/_search' " + "curl --silent --show-error --fail-with-body '${esUrl}/_search' " + "-H 'Content-Type: application/json' " + "-d '{}' ".format(json.dumps(dictionary)) - + "| jq '.hits.total > 0'" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) @@ -178,7 +208,8 @@ let # TODO: extend this test with multiple elasticsearch nodes # and see if the status turns "green". one.wait_until_succeeds( - "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red" + "curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'" + + " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'" ) with subtest("Perform some simple logstash tests"): @@ -189,33 +220,50 @@ let with subtest("Kibana is healthy"): one.wait_for_unit("kibana.service") one.wait_until_succeeds( - "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green" + "curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'" + + " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'" ) with subtest("Metricbeat is running"): one.wait_for_unit("metricbeat.service") with subtest("Metricbeat metrics arrive in elasticsearch"): - one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'") + one.wait_until_succeeds(has_metricbeat()) with subtest("Logstash messages arive in elasticsearch"): - one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0") - one.wait_until_succeeds(total_hits("dragons") + " | grep 0") + one.wait_until_succeeds(expect_hits("flowers")) + one.wait_until_succeeds(expect_no_hits("dragons")) + '' + lib.optionalString (elk ? journalbeat) '' with subtest( "A message logged to the journal is ingested by elasticsearch via journalbeat" ): one.wait_for_unit("journalbeat.service") one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat") one.wait_until_succeeds( - total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0" + expect_hits("Supercalifragilisticexpialidocious") ) - + '' + lib.optionalString (elk ? filebeat) '' + with subtest( + "A message logged to the journal is ingested by elasticsearch via filebeat" + ): + one.wait_for_unit("filebeat.service") + one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat") + one.wait_until_succeeds( + expect_hits("Superdupercalifragilisticexpialidocious") + ) + one.execute( + "echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test" + ) + one.wait_until_succeeds( + expect_hits("SuperdupercalifragilisticexpialidociousIndeed") + ) + '' + '' with subtest("Elasticsearch-curator works"): one.systemctl("stop logstash") one.systemctl("start elasticsearch-curator") one.wait_until_succeeds( - '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^' + '! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^' ) ''; }) { inherit pkgs system; }; @@ -235,7 +283,7 @@ in { # elasticsearch = pkgs.elasticsearch7-oss; # logstash = pkgs.logstash7-oss; # kibana = pkgs.kibana7-oss; - # journalbeat = pkgs.journalbeat7; + # filebeat = pkgs.filebeat7; # metricbeat = pkgs.metricbeat7; # }; unfree = lib.dontRecurseIntoAttrs { @@ -250,7 +298,7 @@ in { elasticsearch = pkgs.elasticsearch7; logstash = pkgs.logstash7; kibana = pkgs.kibana7; - journalbeat = pkgs.journalbeat7; + filebeat = pkgs.filebeat7; metricbeat = pkgs.metricbeat7; }; }; diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix index d838d3b6a39c..50b977723e9c 100644 --- a/nixos/tests/parsedmarc/default.nix +++ b/nixos/tests/parsedmarc/default.nix @@ -4,6 +4,7 @@ { pkgs, ... }@args: let inherit (import ../../lib/testing-python.nix args) makeTest; + inherit (pkgs) lib; dmarcTestReport = builtins.fetchurl { name = "dmarc-test-report"; @@ -54,7 +55,7 @@ in localMail = makeTest { name = "parsedmarc-local-mail"; - meta = with pkgs.lib.maintainers; { + meta = with lib.maintainers; { maintainers = [ talyz ]; }; @@ -83,7 +84,7 @@ in }; }; - services.elasticsearch.package = pkgs.elasticsearch7-oss; + services.elasticsearch.package = pkgs.elasticsearch-oss; environment.systemPackages = [ (sendEmail "dmarc@localhost") @@ -94,6 +95,7 @@ in testScript = { nodes }: let esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value"; in '' parsedmarc.start() parsedmarc.wait_for_unit("postfix.service") @@ -104,11 +106,15 @@ in ) parsedmarc.fail( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) parsedmarc.succeed("send-email") parsedmarc.wait_until_succeeds( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) ''; }; @@ -121,7 +127,7 @@ in in makeTest { name = "parsedmarc-external-mail"; - meta = with pkgs.lib.maintainers; { + meta = with lib.maintainers; { maintainers = [ talyz ]; }; @@ -153,7 +159,7 @@ in }; }; - services.elasticsearch.package = pkgs.elasticsearch7-oss; + services.elasticsearch.package = pkgs.elasticsearch-oss; environment.systemPackages = [ pkgs.jq @@ -201,6 +207,7 @@ in testScript = { nodes }: let esPort = toString nodes.parsedmarc.config.services.elasticsearch.port; + valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value"; in '' mail.start() mail.wait_for_unit("postfix.service") @@ -213,11 +220,15 @@ in ) parsedmarc.fail( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) mail.succeed("send-email") parsedmarc.wait_until_succeeds( - "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'" + "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940" + + " | tee /dev/console" + + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'" ) ''; }; diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix index c728f7375436..cd81975ee445 100644 --- a/pkgs/development/tools/misc/kibana/6.x.nix +++ b/pkgs/development/tools/misc/kibana/6.x.nix @@ -18,12 +18,12 @@ let shas = if enableUnfree then { - x86_64-linux = "1xwklhqxk5rmdrgy2simwvijzq29kyq5w2w3hy53xh2i1zlnyvq3"; - x86_64-darwin = "1qpdn28mrpggd55khzqqld6r89l0hb870rigxcw2i8p2yx3jv106"; + x86_64-linux = "1a501lavxhckb3l93sbrbqyshicwkk6p89frry4x8p037xcfpy0x"; + x86_64-darwin = "0zm45af30shhcg3mdhcma6rms1hyrx62rm5jzwnz9kxv4d30skbw"; } else { - x86_64-linux = "1wpnwal2rq5v2bsp5qil9j6dplif7ql5394sy4ia5ghp2fzifxmf"; - x86_64-darwin = "12z8i0wbw10c097glbpdy350p0h3957433f51qfx2p0ghgkzkhzv"; + x86_64-linux = "0wfdipf21apyily7mvlqgyc7m5jpr96zgrryzwa854z3xb2vw8zg"; + x86_64-darwin = "1nklfx4yz6hsxlljvnvwjy7pncv9mzngl84710xad5jlyras3sdj"; }; in stdenv.mkDerivation rec { diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix index 7a2e0d1d3647..455f95a69eb0 100644 --- a/pkgs/development/tools/misc/kibana/7.x.nix +++ b/pkgs/development/tools/misc/kibana/7.x.nix @@ -4,23 +4,23 @@ , stdenv , makeWrapper , fetchurl -, nodejs-14_x +, nodejs-16_x , coreutils , which }: with lib; let - nodejs = nodejs-14_x; + nodejs = nodejs-16_x; inherit (builtins) elemAt; info = splitString "-" stdenv.hostPlatform.system; arch = elemAt info 0; plat = elemAt info 1; shas = { - x86_64-linux = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1"; - x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc"; - aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8"; + x86_64-linux = "0jivwsrq31n0qfznrsjfsn65sg3wpbd990afn2wzjnj4drq7plz6"; + x86_64-darwin = "02483aqzrccq1x6rwznmcazijdd46yxj9vnbihnvp2xyp3w9as45"; + aarch64-linux = "0iw155gkkl1hshc80lfj95rssg039ig21wz1l3srmmf2x4f934s9"; }; in stdenv.mkDerivation rec { diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix index 1808197498b5..f5e31924791b 100644 --- a/pkgs/misc/logging/beats/6.x.nix +++ b/pkgs/misc/logging/beats/6.x.nix @@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8"; + sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7"; }; goPackagePath = "github.com/elastic/beats"; diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix index b8b82ed4b308..9f5e550d9720 100644 --- a/pkgs/misc/logging/beats/7.x.nix +++ b/pkgs/misc/logging/beats/7.x.nix @@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8"; + sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI="; }; - vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c"; + vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk="; subPackages = [ package ]; @@ -24,7 +24,14 @@ let beat = package: extraArgs: buildGoModule (rec { } // extraArgs); in rec { - filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; }; + filebeat7 = beat "filebeat" { + meta.description = "Lightweight shipper for logfiles"; + buildInputs = [ systemd ]; + tags = [ "withjournald" ]; + postFixup = '' + patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat" + ''; + }; heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; }; metricbeat7 = beat "metricbeat" { meta.description = "Lightweight shipper for metrics"; @@ -47,15 +54,4 @@ rec { PostgreSQL, Redis or Thrift and correlate the messages into transactions. ''; }; - journalbeat7 = beat "journalbeat" { - meta.description = '' - Journalbeat is an open source data collector to read and forward - journal entries from Linuxes with systemd. - ''; - buildInputs = [ systemd.dev ]; - postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in - '' - patchelf --set-rpath ${libPath} "$out/bin/journalbeat" - ''; - }; } diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix index 4b92592f65df..56f0779c833a 100644 --- a/pkgs/servers/search/elasticsearch/6.x.nix +++ b/pkgs/servers/search/elasticsearch/6.x.nix @@ -19,8 +19,8 @@ stdenv.mkDerivation (rec { url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz"; sha256 = if enableUnfree - then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42" - else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39"; + then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd" + else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm"; }; patches = [ ./es-home-6.x.patch ]; diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix index c254b733837b..592cc947a42e 100644 --- a/pkgs/servers/search/elasticsearch/7.x.nix +++ b/pkgs/servers/search/elasticsearch/7.x.nix @@ -18,9 +18,9 @@ let plat = elemAt info 1; shas = { - x86_64-linux = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10"; - x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3"; - aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy"; + x86_64-linux = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c"; + x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs"; + aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq"; }; in stdenv.mkDerivation rec { diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index f71d7b9cc76c..03bb24d9a390 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -38,8 +38,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy" - else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv" + if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p" + else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -55,8 +55,8 @@ in src = fetchurl { url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip"; sha256 = - if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3" - else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53" + if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir" + else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -72,8 +72,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855" - else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37" + if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x" + else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -89,8 +89,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6" - else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im" + if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii" + else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -106,8 +106,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs" - else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll" + if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4" + else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -123,8 +123,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip"; sha256 = - if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs" - else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x" + if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml" + else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -140,8 +140,8 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip"; sha256 = - if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7" - else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk" + if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1" + else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv" else throw "unsupported version ${version} for plugin ${pluginName}"; }; meta = with lib; { @@ -157,24 +157,24 @@ in pluginName = "search-guard"; version = # https://docs.search-guard.com/latest/search-guard-versions - if esVersion == "7.11.1" then "${esVersion}-50.0.0" - else if esVersion == "6.8.3" then "${esVersion}-25.5" + if esVersion == "7.16.1" then "${esVersion}-52.5.0" + else if esVersion == "6.8.21" then "${esVersion}-25.6" else throw "unsupported version ${esVersion} for plugin ${pluginName}"; src = - if esVersion == "7.11.1" then + if esVersion == "7.16.1" then fetchurl { url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip"; - sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm"; + sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb"; } - else if esVersion == "6.8.3" then + else if esVersion == "6.8.21" then fetchurl { - url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip"; - sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb"; + url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip"; + sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2"; } else throw "unsupported version ${version} for plugin ${pluginName}"; meta = with lib; { homepage = "https://search-guard.com"; - description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. "; + description = "Elasticsearch plugin that offers encryption, authentication, and authorisation."; license = licenses.asl20; }; }; diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix index b35e5a4aea43..0b3e17818dcd 100644 --- a/pkgs/tools/misc/logstash/6.x.nix +++ b/pkgs/tools/misc/logstash/6.x.nix @@ -17,8 +17,8 @@ let this = stdenv.mkDerivation rec { url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz"; sha256 = if enableUnfree - then "00pwi7clgdflzzg15bh3y30gzikvvy7p5fl88fww7xhhy47q8053" - else "0spxgqsyh72n0l0xh6rljp0lbqz46xmr02sqz25ybycr4qkxdhgk"; + then "0hij1byw5b3xmk3vshr9p7gxwbjrywr7ylps05ydc2dmnz8q2a79" + else "1fa236pvhj7spys54nqi3k64rwzf6zi6gaccmqg4p4sh92jzsybv"; }; dontBuild = true; diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix index 1e69fbc976d9..636c380817ce 100644 --- a/pkgs/tools/misc/logstash/7.x.nix +++ b/pkgs/tools/misc/logstash/7.x.nix @@ -17,14 +17,14 @@ let shas = if enableUnfree then { - x86_64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p"; - x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm"; - aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8"; + x86_64-linux = "1vm53alq9q1qy2jcsjg9z339xrkac5r9qqpdafp53ny4zsv1n7vj"; + x86_64-darwin = "0hhjyl04h3gd66rdk22272rj419br4v2i59lyrmaj6hmnsqbv968"; + aarch64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p"; } else { - x86_64-linux = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3"; - x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs"; - aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2"; + x86_64-linux = "1f3659vcgczm7v03q3fvsmp1ndp6wm3i7r2b2vbl4xq7hf9v7azk"; + x86_64-darwin = "10zw9qc0lc0x9in0nkxc1aiazhyd69l8sya2ni46ivyyjwf0sqsn"; + aarch64-linux = "1czhgmky2zf3mqykn5ww4257yfhd36mi4x6dq569ymly83pivf8v"; }; this = stdenv.mkDerivation rec { version = elk7Version; diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 222c6ac0474f..672fdf21b025 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -384,6 +384,7 @@ mapAliases ({ jbuilder = dune_1; # added 2018-09-09 jikes = throw "jikes was deprecated on 2019-10-07: abandoned by upstream"; joseki = apache-jena-fuseki; # added 2016-02-28 + journalbeat7 = throw "journalbeat has been removed upstream. Use filebeat with the journald input instead."; jvmci8 = throw "graalvm8 and its tools were deprecated in favor of graalvm8-ce"; # added 2021-10-15 json_glib = json-glib; # added 2018-02-25 kafkacat = kcat; # added 2021-10-07 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 8d98f53a3318..9c117ca4d712 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2140,8 +2140,7 @@ with pkgs; filebeat7 heartbeat7 metricbeat7 - packetbeat7 - journalbeat7; + packetbeat7; filebeat = filebeat6; heartbeat = heartbeat6; @@ -5000,8 +4999,8 @@ with pkgs; # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. # When updating make sure to update all plugins or they will break! - elk6Version = "6.8.3"; - elk7Version = "7.11.1"; + elk6Version = "6.8.21"; + elk7Version = "7.16.1"; elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { util-linux = util-linuxMinimal;