diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 2a1a4cba2956..e37bae7479b4 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -1,12 +1,15 @@ +# To run the test on the unfree ELK use the folllowing command: +# cd path/to/nixpkgs +# NIXPKGS_ALLOW_UNFREE=1 nix-build -A nixosTests.elk.unfree.ELK-6 + { system ? builtins.currentSystem, config ? {}, pkgs ? import ../.. { inherit system config; }, - enableUnfree ? false - # To run the test on the unfree ELK use the folllowing command: - # NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true }: let + inherit (pkgs) lib; + esUrl = "http://localhost:9200"; mkElkTest = name : elk : @@ -215,38 +218,40 @@ let '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^' ) ''; - }) {}; -in pkgs.lib.mapAttrs mkElkTest { - ELK-6 = - if enableUnfree - then { + }) { inherit pkgs system; }; +in { + ELK-6 = mkElkTest { + name = "elk-6-oss"; + elasticsearch = pkgs.elasticsearch6-oss; + logstash = pkgs.logstash6-oss; + kibana = pkgs.kibana6-oss; + journalbeat = pkgs.journalbeat6; + metricbeat = pkgs.metricbeat6; + }; + # We currently only package upstream binaries. + # Feel free to package an SSPL licensed source-based package! + # ELK-7 = mkElkTest { + # name = "elk-7"; + # elasticsearch = pkgs.elasticsearch7-oss; + # logstash = pkgs.logstash7-oss; + # kibana = pkgs.kibana7-oss; + # journalbeat = pkgs.journalbeat7; + # metricbeat = pkgs.metricbeat7; + # }; + unfree = lib.dontRecurseIntoAttrs { + ELK-6 = mkElkTest "elk-6" { elasticsearch = pkgs.elasticsearch6; logstash = pkgs.logstash6; kibana = pkgs.kibana6; journalbeat = pkgs.journalbeat6; metricbeat = pkgs.metricbeat6; - } - else { - elasticsearch = pkgs.elasticsearch6-oss; - logstash = pkgs.logstash6-oss; - kibana = pkgs.kibana6-oss; - journalbeat = pkgs.journalbeat6; - metricbeat = pkgs.metricbeat6; }; - ELK-7 = - if enableUnfree - then { + ELK-7 = mkElkTest "elk-7" { elasticsearch = pkgs.elasticsearch7; logstash = pkgs.logstash7; kibana = pkgs.kibana7; journalbeat = pkgs.journalbeat7; metricbeat = pkgs.metricbeat7; - } - else { - elasticsearch = pkgs.elasticsearch7-oss; - logstash = pkgs.logstash7-oss; - kibana = pkgs.kibana7-oss; - journalbeat = pkgs.journalbeat7; - metricbeat = pkgs.metricbeat7; }; + }; } diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix index a2b6f740db4c..7a2e0d1d3647 100644 --- a/pkgs/development/tools/misc/kibana/7.x.nix +++ b/pkgs/development/tools/misc/kibana/7.x.nix @@ -4,32 +4,27 @@ , stdenv , makeWrapper , fetchurl -, nodejs-10_x +, nodejs-14_x , coreutils , which }: with lib; let - nodejs = nodejs-10_x; + nodejs = nodejs-14_x; inherit (builtins) elemAt; info = splitString "-" stdenv.hostPlatform.system; arch = elemAt info 0; plat = elemAt info 1; shas = - if enableUnfree - then { - x86_64-linux = "sha256-lTPBppKm51zgKSQtSdO0PgZ/aomvaStwqwYYGNPY4Bo="; - x86_64-darwin = "sha256-d7xHmoASiywDlZCJX/CfUX1VIi4iOcDrqvK0su54MJc="; - } - else { - x86_64-linux = "sha256-+pkKpiXBpLHs72KKNtMJbqipw6eu5XC1xu/iLFCHGRQ="; - x86_64-darwin = "sha256-CyJ5iRXaPgXO2lyy+E24OcGtb9V3e1gMZRIu25bVyzk="; + { + x86_64-linux = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1"; + x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc"; + aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8"; }; -in -stdenv.mkDerivation rec { - pname = "kibana${optionalString (!enableUnfree) "-oss"}"; +in stdenv.mkDerivation rec { + pname = "kibana"; version = elk7Version; src = fetchurl { @@ -58,7 +53,7 @@ stdenv.mkDerivation rec { meta = { description = "Visualize logs and time-stamped data"; homepage = "http://www.elasticsearch.org/overview/kibana"; - license = if enableUnfree then licenses.elastic else licenses.asl20; + license = licenses.elastic; maintainers = with maintainers; [ offline basvandijk ]; platforms = with platforms; unix; }; diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix index 99a79fecedc5..b8b82ed4b308 100644 --- a/pkgs/misc/logging/beats/7.x.nix +++ b/pkgs/misc/logging/beats/7.x.nix @@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "sha256-zr0a0LBR4G9okS2pUixDYtYZ0yCp4G6j08jx/zlIKOA="; + sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8"; }; - vendorSha256 = "sha256-xmw432vY1T2EixkDcXdGrnMdc8fYOI4R2lEjbkav3JQ="; + vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c"; subPackages = [ package ]; diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix index de194bcc7d29..c254b733837b 100644 --- a/pkgs/servers/search/elasticsearch/7.x.nix +++ b/pkgs/servers/search/elasticsearch/7.x.nix @@ -1,5 +1,4 @@ { elk7Version -, enableUnfree ? true , lib , stdenv , fetchurl @@ -18,19 +17,15 @@ let arch = elemAt info 0; plat = elemAt info 1; shas = - if enableUnfree - then { - x86_64-linux = "sha256-O3rjtvXyJI+kRBqiz2U2OMkCIQj4E+AIHaE8N4o14R4="; - x86_64-darwin = "sha256-AwuY2yMxf+v7U5/KD3Cf+Hv6ijjySEyj6pzF3RCsg24="; - } - else { - x86_64-linux = "sha256-cJrdkFIFgAI6wfQh34Z8yFuLrOCOKzgOsWZhU3S/3NQ="; - x86_64-darwin = "sha256-OhMVOdXei9D9cH+O5tBhdKvZ05TsImjMqUUsucRyWMo="; + { + x86_64-linux = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10"; + x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3"; + aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy"; }; in -stdenv.mkDerivation (rec { +stdenv.mkDerivation rec { + pname = "elasticsearch"; version = elk7Version; - pname = "elasticsearch${optionalString (!enableUnfree) "-oss"}"; src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}-${plat}-${arch}.tar.gz"; @@ -49,9 +44,11 @@ stdenv.mkDerivation (rec { "ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/*\"" ''; - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ jre_headless util-linux ] - ++ optional enableUnfree zlib; + nativeBuildInputs = [ autoPatchelfHook makeWrapper ]; + + buildInputs = [ jre_headless util-linux zlib ]; + + runtimeDependencies = [ zlib ]; installPhase = '' mkdir -p $out @@ -69,22 +66,12 @@ stdenv.mkDerivation (rec { wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}" ''; - passthru = { inherit enableUnfree; }; + passthru = { enableUnfree = true; }; meta = { description = "Open Source, Distributed, RESTful Search Engine"; - license = if enableUnfree then licenses.elastic else licenses.asl20; + license = licenses.elastic; platforms = platforms.unix; maintainers = with maintainers; [ apeschar basvandijk ]; }; -} // optionalAttrs enableUnfree { - dontPatchELF = true; - nativeBuildInputs = [ makeWrapper autoPatchelfHook ]; - runtimeDependencies = [ zlib ]; - postFixup = '' - for exe in $(find $out/modules/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do - echo "patching $exe..." - patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe" - done - ''; -}) +} diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index 39af6976bf9b..f71d7b9cc76c 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -4,13 +4,14 @@ let esVersion = elasticsearch.version; esPlugin = - a@{ pluginName - , installPhase ? '' + a@{ + pluginName, + installPhase ? '' mkdir -p $out/config mkdir -p $out/plugins - ln -s ${elasticsearch}/lib $out/lib + ln -s ${elasticsearch}/lib ${elasticsearch}/modules $out ES_HOME=$out ${elasticsearch}/bin/elasticsearch-plugin install --batch -v file://$src - rm $out/lib + rm $out/lib $out/modules '' , ... }: @@ -37,7 +38,7 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.10.2" then "sha256-HXNJy8WPExPeh5afjdLEFg+0WX0LYI/kvvaLGVUke5E=" + if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy" else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -54,7 +55,7 @@ in src = fetchurl { url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip"; sha256 = - if version == "7.10.2" then "sha256-mW4YNZ20qatyfHCDAmod/gVmkPYh15NrsYPgiBy1/T8=" + if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3" else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -71,7 +72,7 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.10.2" then "sha256-PjA/pwoulkD2d6sHKqzcYxQpb1aS68/l047z5JTcV3Y=" + if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855" else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -88,7 +89,7 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.10.2" then "sha256-yvxSkVyZDWeu7rcxxq1+IVsljZQKgWEURiXY9qycK1s=" + if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6" else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -105,7 +106,7 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip"; sha256 = - if version == "7.10.2" then "sha256-yOMiYJ2c/mcLDcTA99YrpQBiEBAa/mLtTqJlqTJ5tBc=" + if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs" else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -122,7 +123,7 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip"; sha256 = - if version == "7.10.2" then "sha256-fN2RQsY9OACE71pIw87XVJo4c3sUu/6gf/6wUt7ZNIE=" + if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs" else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -139,7 +140,7 @@ in src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip"; sha256 = - if version == "7.10.2" then "sha256-JdWt5LzSbs0MIEuLJIE1ceTnNeTYI5Jt2N0Xj7OBO6g=" + if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7" else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk" else throw "unsupported version ${version} for plugin ${pluginName}"; }; @@ -150,30 +151,31 @@ in }; }; - search-guard = - let - majorVersion = lib.head (builtins.splitVersion esVersion); - in - esPlugin rec { - pluginName = "search-guard"; - version = - # https://docs.search-guard.com/latest/search-guard-versions - if esVersion == "7.10.2" then "7.10.1-49.3.0" - else if esVersion == "6.8.3" then "${esVersion}-25.5" - else throw "unsupported version ${esVersion} for plugin ${pluginName}"; - src = fetchurl { - url = - if version == "7.10.1-49.3.0" then "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip" - else "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip"; - sha256 = - if version == "7.10.1-49.3.0" then "sha256-vKH2+c+7WlncgljrvYH9lAqQTKzg9l0ABZ23Q/xdoK4=" - else if version == "6.8.3-25.5" then "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb" - else throw "unsupported version ${version} for plugin ${pluginName}"; - }; - meta = with lib; { - homepage = "https://search-guard.com"; - description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. "; - license = licenses.asl20; - }; + search-guard = let + majorVersion = lib.head (builtins.splitVersion esVersion); + in esPlugin rec { + pluginName = "search-guard"; + version = + # https://docs.search-guard.com/latest/search-guard-versions + if esVersion == "7.11.1" then "${esVersion}-50.0.0" + else if esVersion == "6.8.3" then "${esVersion}-25.5" + else throw "unsupported version ${esVersion} for plugin ${pluginName}"; + src = + if esVersion == "7.11.1" then + fetchurl { + url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip"; + sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm"; + } + else if esVersion == "6.8.3" then + fetchurl { + url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip"; + sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb"; + } + else throw "unsupported version ${version} for plugin ${pluginName}"; + meta = with lib; { + homepage = "https://search-guard.com"; + description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. "; + license = licenses.asl20; }; + }; } diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix index c0c67b19b10a..1e69fbc976d9 100644 --- a/pkgs/tools/misc/logstash/7.x.nix +++ b/pkgs/tools/misc/logstash/7.x.nix @@ -17,17 +17,20 @@ let shas = if enableUnfree then { - x86_64-linux = "sha256-5qv4fbFpLf6aduD7wyxXQ6FsCeUqrszRisNBx44vbMY="; - x86_64-darwin = "sha256-7H+Xpo8qF1ZZMkR5n92PVplEN4JsBEYar91zHQhE+Lo="; + x86_64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p"; + x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm"; + aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8"; } else { - x86_64-linux = "sha256-jiV2yGPwPgZ5plo3ftImVDLSOsk/XBzFkeeALSObLhU="; - x86_64-darwin = "sha256-UYG+GGr23eAc2GgNX/mXaGU0WKMjiQMPpD1wUvAVz0A="; + x86_64-linux = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3"; + x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs"; + aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2"; }; this = stdenv.mkDerivation rec { version = elk7Version; pname = "logstash${optionalString (!enableUnfree) "-oss"}"; + src = fetchurl { url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}-${plat}-${arch}.tar.gz"; sha256 = shas.${stdenv.hostPlatform.system} or (throw "Unknown architecture"); diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 37455858c0a0..38189f6ae296 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -213,6 +213,7 @@ mapAliases ({ ec2_ami_tools = ec2-ami-tools; # added 2021-10-08 ec2_api_tools = ec2-api-tools; # added 2021-10-08 elasticmq = throw "elasticmq has been removed in favour of elasticmq-server-bin"; # added 2021-01-17 + elasticsearch7-oss = throw "elasticsearch7-oss has been removed, as the distribution is no longer provided by upstream. https://github.com/NixOS/nixpkgs/pull/114456"; # added 2021-06-09 emacsPackagesGen = emacsPackagesFor; # added 2018-08-18 emacsPackagesNgGen = emacsPackagesFor; # added 2018-08-18 emacsPackagesNgFor = emacsPackagesFor; # added 2019-08-07 @@ -376,6 +377,7 @@ mapAliases ({ json_glib = json-glib; # added 2018-02-25 kdecoration-viewer = throw "kdecoration-viewer has been removed from nixpkgs, as there is no upstream activity"; # 2020-06-16 k9copy = throw "k9copy has been removed from nixpkgs, as there is no upstream activity"; # 2020-11-06 + kibana7-oss = throw "kibana7-oss has been removed, as the distribution is no longer provided by upstream. https://github.com/NixOS/nixpkgs/pull/114456"; # added 2021-06-09 kodiGBM = kodi-gbm; kodiPlain = kodi; kodiPlainWayland = kodi-wayland; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c26fa534310d..b7caf86fe49f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -4863,7 +4863,7 @@ with pkgs; # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. # When updating make sure to update all plugins or they will break! elk6Version = "6.8.3"; - elk7Version = "7.10.2"; + elk7Version = "7.11.1"; elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { util-linux = util-linuxMinimal; @@ -4878,11 +4878,6 @@ with pkgs; util-linux = util-linuxMinimal; jre_headless = jdk11_headless; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731 }; - elasticsearch7-oss = callPackage ../servers/search/elasticsearch/7.x.nix { - enableUnfree = false; - util-linux = util-linuxMinimal; - jre_headless = jdk11_headless; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731 - }; elasticsearch = elasticsearch6; elasticsearch-oss = elasticsearch6-oss; @@ -4895,7 +4890,7 @@ with pkgs; elasticsearch = elasticsearch6-oss; }; elasticsearch7Plugins = elasticsearchPlugins.override { - elasticsearch = elasticsearch7-oss; + elasticsearch = elasticsearch7; }; elasticsearch-curator = callPackage ../tools/admin/elasticsearch-curator { @@ -6704,9 +6699,6 @@ with pkgs; enableUnfree = false; }; kibana7 = callPackage ../development/tools/misc/kibana/7.x.nix { }; - kibana7-oss = callPackage ../development/tools/misc/kibana/7.x.nix { - enableUnfree = false; - }; kibana = kibana6; kibana-oss = kibana6-oss;